remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-17 22:14:09 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'main' into fts-quote

Browse source
This commit is contained in:
Simon Willison 2021-08-18 11:40:46 -07:00 • committed by GitHub
commit af989af658
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 5850 additions and 571 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/publish.yml vendored
View file

@ -10,7 +10,7 @@ jobs:
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
os: [ubuntu-latest, macos-latest, windows-latest]
os: [ubuntu-latest, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}

27
.github/workflows/spellcheck.yml vendored Normal file
View file

@ -0,0 +1,27 @@
name: Check spelling in documentation
on: [push, pull_request]
jobs:
spellcheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: 3.9
- uses: actions/cache@v2
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: |
pip install -e '.[docs]'
- name: Check spelling
run: |
codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt

41
.github/workflows/test-coverage.yml vendored Normal file
View file

@ -0,0 +1,41 @@
name: Calculate test coverage
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Check out repo
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.9
- uses: actions/cache@v2
name: Configure pip caching
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -e .[test]
python -m pip install pytest-cov
- name: Run tests
run: |-
ls -lah
pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term
ls -lah
- name: Upload coverage report
uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: coverage.xml

6
.github/workflows/test.yml vendored
View file

@ -25,12 +25,16 @@ jobs:
${{ runner.os }}-pip-
- name: Install dependencies
run: |
pip install -e '.[test]'
pip install -e '.[test,mypy,flake8]'
- name: Optionally install numpy
if: matrix.numpy == 1
run: pip install numpy
- name: Run tests
run: |
pytest
- name: run mypy
run: mypy sqlite_utils
- name: run flake8
run: flake8
- name: Check formatting
run: black . --check

12
.readthedocs.yaml Normal file
View file

@ -0,0 +1,12 @@
version: 2
sphinx:
configuration: docs/conf.py
python:
version: "3.8"
install:
- method: pip
path: .
extra_requirements:
- docs

11
README.md
View file

@ -1,10 +1,11 @@
# sqlite-utils
[![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/)
[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html)
[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html)
[![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/)
[![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest)
[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest)
[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable)
[![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils)
[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE)
Python CLI utility and library for manipulating SQLite databases.
@ -56,7 +57,11 @@ You can import JSON data into a new database table like this:
Or for data in a CSV file:
$ sqlite-utils insert dogs.db dogs docs.csv --csv
$ sqlite-utils insert dogs.db dogs dogs.csv --csv
`sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command:
$ cat dogs.csv | sqlite-utils memory - "select name, age from dogs"
See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands.

8
codecov.yml Normal file
View file

@ -0,0 +1,8 @@
coverage:
status:
project:
default:
informational: true
patch:
default:
informational: true

2
docs/Makefile
View file

@ -20,4 +20,4 @@ help:
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
livehtml:
sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils

408
docs/changelog.rst
View file

@ -2,6 +2,184 @@
Changelog
===========
.. _v3_15.1:
3.15.1 (2021-08-10)
-------------------
- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`)
- New :ref:`reference` documentation page, powered by those docstrings.
- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`)
- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`)
.. _v3_15:
3.15 (2021-08-09)
-----------------
- ``sqlite-utils insert --flatten`` option for :ref:`flattening nested JSON objects <cli_inserting_data_flatten>` to create tables with column names like ``topkey_nestedkey``. (:issue:`310`)
- Fixed several spelling mistakes in the documentation, spotted `using codespell <https://til.simonwillison.net/python/codespell>`__.
- Errors that occur while using the ``sqlite-utils`` CLI tool now show the responsible SQL and query parameters, if possible. (:issue:`309`)
.. _v3_14:
3.14 (2021-08-02)
-----------------
This release introduces the new :ref:`sqlite-utils convert command <cli_convert>` (:issue:`251`) and corresponding :ref:`table.convert(...) <python_api_convert>` Python method (:issue:`302`). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns.
This command-line example uses the Python standard library `textwrap module <https://docs.python.org/3/library/textwrap.html>`__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters::
$ sqlite-utils convert content.db articles content \
'"\n".join(textwrap.wrap(value, 100))' \
--import=textwrap
The same operation in Python code looks like this:
.. code-block:: python
import sqlite_utils, textwrap
db = sqlite_utils.Database("content.db")
db["articles"].convert("content", lambda v: "\n".join(textwrap.wrap(v, 100)))
See the full documentation for the :ref:`sqlite-utils convert command <cli_convert>` and the :ref:`table.convert(...) <python_api_convert>` Python method for more details.
Also in this release:
- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (:issue:`305`)
- New ``--silent`` option for the :ref:`sqlite-utils insert-files command <cli_insert_files>` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (:issue:`301`)
.. _v3_13:
3.13 (2021-07-24)
-----------------
- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (:issue:`299`)
- ``sqlite-utils memory --help`` now describes the ``--schema`` option.
.. _v3_12:
3.12 (2021-06-25)
-----------------
- New :ref:`db.query(sql, params) <python_api_query>` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (:issue:`290`)
- This project now uses ``flake8`` and has started to use ``mypy``. (:issue:`291`)
- New documentation on :ref:`contributing <contributing>` to this project. (:issue:`292`)
.. _v3_11:
3.11 (2021-06-20)
-----------------
- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (:issue:`288`)
- Added :ref:`installation instructions <installation>`. (:issue:`286`)
.. _v3_10:
3.10 (2021-06-19)
-----------------
This release introduces the ``sqlite-utils memory`` command, which can be used to load CSV or JSON data into a temporary in-memory database and run SQL queries (including joins across multiple files) directly against that data.
Also new: ``sqlite-utils insert --detect-types``, ``sqlite-utils dump``, ``table.use_rowid`` plus some smaller fixes.
sqlite-utils memory
~~~~~~~~~~~~~~~~~~~
This example of ``sqlite-utils memory`` retrieves information about the all of the repositories in the `Dogsheep <https://github.com/dogsheep>`__ organization on GitHub using `this JSON API <https://api.github.com/users/dogsheep/repos>`__, sorts them by their number of stars and outputs a table of the top five (using ``-t``)::
$ curl -s 'https://api.github.com/users/dogsheep/repos' \
| sqlite-utils memory - '
select full_name, forks_count, stargazers_count
from stdin order by stargazers_count desc limit 5
' -t
full_name forks_count stargazers_count
--------------------------------- ------------- ------------------
dogsheep/twitter-to-sqlite 12 225
dogsheep/github-to-sqlite 14 139
dogsheep/dogsheep-photos 5 116
dogsheep/dogsheep.github.io 7 90
dogsheep/healthkit-to-sqlite 4 85
The tool works against files on disk as well. This example joins data from two CSV files::
$ cat creatures.csv
species_id,name
1,Cleo
2,Bants
2,Dori
2,Azi
$ cat species.csv
id,species_name
1,Dog
2,Chicken
$ sqlite-utils memory species.csv creatures.csv '
select * from creatures join species on creatures.species_id = species.id
'
[{"species_id": 1, "name": "Cleo", "id": 1, "species_name": "Dog"},
{"species_id": 2, "name": "Bants", "id": 2, "species_name": "Chicken"},
{"species_id": 2, "name": "Dori", "id": 2, "species_name": "Chicken"},
{"species_id": 2, "name": "Azi", "id": 2, "species_name": "Chicken"}]
Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv`` file becomes the ``creatures`` table and the output is JSON, the default output format.
You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables.
Full documentation of this new feature is available in :ref:`cli_memory`. (:issue:`272`)
sqlite-utils insert \-\-detect-types
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The :ref:`sqlite-utils insert <cli_inserting_data>` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (:issue:`282`)
Other changes
~~~~~~~~~~~~~
- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (:issue:`284`)
- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (:issue:`285`)
- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (:issue:`274`)
- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (:issue:`276`)
- Now using `pytest-cov <https://pytest-cov.readthedocs.io/>`__ and `Codecov <https://about.codecov.io/>`__ to track test coverage - currently at 96%. (:issue:`275`)
- SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors.
.. _v3_9_1:
3.9.1 (2021-06-12)
------------------
- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (:issue:`271`)
.. _v3_9:
3.9 (2021-06-11)
----------------
- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)<cli_schema>`. (:issue:`268`)
- ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) <python_api_schema>`.
.. _v3_8:
3.8 (2021-06-02)
----------------
- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (:issue:`263`)
- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (:issue:`261`)
.. _v3_7:
3.7 (2021-05-28)
----------------
- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (:issue:`240`)
- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (:issue:`238`)
- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (:issue:`237`)
- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (:issue:`237`)
- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (:issue:`257`)
- Suggest ``--alter`` if an error occurs caused by a missing column. (:issue:`259`)
- Support creating indexes with columns in descending order, see :ref:`API documentation <python_api_create_index>` and :ref:`CLI documentation <cli_create_index>`. (:issue:`260`)
- Correctly handle CSV files that start with a UTF-8 BOM. (:issue:`250`)
.. _v3_6:
3.6 (2021-02-18)
@ -9,73 +187,73 @@
This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 <https://docs.datasette.io/en/stable/changelog.html#v0-55>`__.
- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation <python_api_attach>`. (`#113 <https://github.com/simonw/sqlite-utils/issues/113>`__)
- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation <cli_attach>`. (`#236 <https://github.com/simonw/sqlite-utils/issues/236>`__)
- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation <python_api_attach>`. (:issue:`113`)
- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation <cli_query_attach>`. (:issue:`236`)
.. _v3_5:
3.5 (2021-02-14)
----------------
- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 <https://github.com/simonw/sqlite-utils/issues/230>`__)
- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 <https://github.com/simonw/sqlite-utils/issues/231>`__)
- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 <https://github.com/simonw/sqlite-utils/issues/228>`__)
- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the fix. (`#234 <https://github.com/simonw/sqlite-utils/issues/234>`__)
- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 <https://github.com/simonw/sqlite-utils/issues/229>`__)
- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the Windows test fixes. (`#232 <https://github.com/simonw/sqlite-utils/issues/232>`__)
- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`230`)
- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (:issue:`231`)
- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (:issue:`228`)
- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the fix. (:issue:`234`)
- Fixed bug importing CSV files with columns containing more than 128KB of data. (:issue:`229`)
- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven <https://github.com/nieuwenhoven>`__ for the Windows test fixes. (:issue:`232`)
.. _v3_4_1:
3.4.1 (2021-02-05)
------------------
- Fixed a code import bug that slipped in to 3.4. (`#226 <https://github.com/simonw/sqlite-utils/issues/226>`__)
- Fixed a code import bug that slipped in to 3.4. (:issue:`226`)
.. _v3_4:
3.4 (2021-02-05)
----------------
- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 <https://github.com/simonw/sqlite-utils/issues/223>`__)
- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`223`)
.. _v3_3:
3.3 (2021-01-17)
----------------
- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 <https://github.com/simonw/sqlite-utils/issues/222>`__)
- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (:issue:`222`)
.. _v3_2_1:
3.2.1 (2021-01-12)
------------------
- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 <https://github.com/simonw/sqlite-utils/issues/221>`__)
- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (:issue:`221`)
.. _v3_2:
3.2 (2021-01-03)
----------------
This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 <https://github.com/simonw/sqlite-utils/issues/212>`__)
This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (:issue:`212`)
- ``table.enable_counts()`` method for enabling these triggers on a specific table.
- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 <https://github.com/simonw/sqlite-utils/issues/213>`__)
- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 <https://github.com/simonw/sqlite-utils/issues/214>`__)
- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 <https://github.com/simonw/sqlite-utils/issues/218>`__)
- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 <https://github.com/simonw/sqlite-utils/issues/215>`__)
- ``db.enable_counts()`` method for enabling triggers on every table in the database. (:issue:`213`)
- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (:issue:`214`)
- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (:issue:`218`)
- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (:issue:`215`)
- ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers.
- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 <https://github.com/simonw/sqlite-utils/issues/219>`__)
- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 <https://github.com/simonw/sqlite-utils/issues/217>`__)
- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 <https://github.com/simonw/sqlite-utils/issues/211>`__, `#216 <https://github.com/simonw/sqlite-utils/issues/216>`__)
- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 <https://github.com/simonw/sqlite-utils/issues/206>`__)
- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (:issue:`219`)
- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (:issue:`217`)
- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (:issue:`211`, :issue:`216`)
- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (:issue:`206`)
.. _v3_1_1:
3.1.1 (2021-01-01)
------------------
- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 <https://github.com/simonw/sqlite-utils/issues/209>`__)
- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (:issue:`209`)
- Documentation now lives on https://sqlite-utils.datasette.io/
- README now includes ``brew install sqlite-utils`` installation method.
@ -84,7 +262,7 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi
3.1 (2020-12-12)
----------------
- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 <https://github.com/simonw/sqlite-utils/issues/207>`__)
- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (:issue:`207`)
- New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`.
- The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 <https://github.com/simonw/sqlite-utils/pull/204>`__)
@ -93,28 +271,28 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi
3.0 (2020-11-08)
----------------
This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 <https://github.com/simonw/sqlite-utils/issues/192>`__)
This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (:issue:`192`)
The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 <https://github.com/simonw/sqlite-utils/issues/197>`__)
The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (:issue:`197`)
The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are:
- The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead.
- The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``.
- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 <https://github.com/simonw/sqlite-utils/issues/198>`__)
- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (:issue:`198`)
- The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples.
Also in this release:
- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 <https://github.com/simonw/sqlite-utils/issues/193>`__)
- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 <https://github.com/simonw/sqlite-utils/issues/196>`__)
- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (:issue:`193`)
- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (:issue:`196`)
- The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`.
- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 <https://github.com/simonw/sqlite-utils/issues/200>`__)
- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (:issue:`200`)
Changes since the 3.0a0 alpha release:
- The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option.
- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 <https://github.com/simonw/sqlite-utils/issues/201>`__)
- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (:issue:`201`)
.. _v2_23:
@ -122,30 +300,30 @@ Changes since the 3.0a0 alpha release:
-----------------
- ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 <https://github.com/simonw/sqlite-utils/pull/189>`__)
- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 <https://github.com/simonw/sqlite-utils/issues/173>`__)
- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 <https://github.com/simonw/sqlite-utils/issues/191>`__)
- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (:issue:`173`)
- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (:issue:`191`)
.. _v2_22:
2.22 (2020-10-16)
-----------------
- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 <https://github.com/simonw/sqlite-utils/issues/182>`__)
- The ``--load-extension`` option is now available to many more commands. (`#137 <https://github.com/simonw/sqlite-utils/issues/137>`__)
- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 <https://github.com/simonw/sqlite-utils/issues/136>`__)
- Tests now also run against Python 3.9. (`#184 <https://github.com/simonw/sqlite-utils/issues/184>`__)
- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 <https://github.com/simonw/sqlite-utils/issues/181>`__)
- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (:issue:`182`)
- The ``--load-extension`` option is now available to many more commands. (:issue:`137`)
- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (:issue:`136`)
- Tests now also run against Python 3.9. (:issue:`184`)
- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (:issue:`181`)
.. _v2_21:
2.21 (2020-09-24)
-----------------
- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 <https://github.com/simonw/sqlite-utils/issues/172>`__)
- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (:issue:`172`)
- ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one.
- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 <https://github.com/simonw/sqlite-utils/issues/175>`__)
- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 <https://github.com/simonw/sqlite-utils/issues/176>`__)
- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 <https://github.com/simonw/sqlite-utils/issues/177>`__)
- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (:issue:`175`)
- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (:issue:`176`)
- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (:issue:`177`)
- The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations.
.. _v2_20:
@ -153,7 +331,7 @@ Changes since the 3.0a0 alpha release:
2.20 (2020-09-22)
-----------------
This release introduces two key new capabilities: **transform** (`#114 <https://github.com/simonw/sqlite-utils/issues/114>`__) and **extract** (`#42 <https://github.com/simonw/sqlite-utils/issues/42>`__).
This release introduces two key new capabilities: **transform** (:issue:`114`) and **extract** (:issue:`42`).
Transform
~~~~~~~~~
@ -174,7 +352,7 @@ The Python library :ref:`extract() documentation <python_api_extract>` describes
Other changes
~~~~~~~~~~~~~
- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 <https://github.com/simonw/sqlite-utils/issues/162>`__)
- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (:issue:`162`)
- The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`.
.. _v2_19:
@ -182,31 +360,31 @@ Other changes
2.19 (2020-09-20)
-----------------
- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 <https://github.com/simonw/sqlite-utils/issues/157>`__)
- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 <https://github.com/simonw/sqlite-utils/issues/160>`__)
- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 <https://github.com/simonw/sqlite-utils/issues/112>`__)
- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (:issue:`157`)
- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (:issue:`160`)
- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (:issue:`112`)
.. _v2_18:
2.18 (2020-09-08)
-----------------
- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 <https://github.com/simonw/sqlite-utils/issues/155>`__)
- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 <https://github.com/simonw/sqlite-utils/issues/155>`__)
- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (:issue:`155`)
- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (:issue:`155`)
- ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem.
- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 <https://github.com/simonw/sqlite-utils/issues/145>`__)
- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (:issue:`145`)
.. _v2_17:
2.17 (2020-09-07)
-----------------
This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 <https://github.com/simonw/sqlite-utils/issues/149>`__)
This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (:issue:`149`)
- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 <https://github.com/simonw/sqlite-utils/issues/152>`__)
- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 <https://github.com/simonw/sqlite-utils/issues/153>`__)
- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 <https://github.com/simonw/sqlite-utils/issues/150>`__)
- Neater indentation for schema SQL. (`#148 <https://github.com/simonw/sqlite-utils/issues/148>`__)
- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (:issue:`152`)
- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (:issue:`153`)
- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (:issue:`150`)
- Neater indentation for schema SQL. (:issue:`148`)
- Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``.
.. _v2_16_1:
@ -214,23 +392,23 @@ This release handles a bug where replacing rows in FTS tables could result in gr
2.16.1 (2020-08-28)
-------------------
- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 <https://github.com/simonw/sqlite-utils/issues/139>`__)
- Continuous Integration is now powered by GitHub Actions. (`#143 <https://github.com/simonw/sqlite-utils/issues/143>`__)
- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (:issue:`139`)
- Continuous Integration is now powered by GitHub Actions. (:issue:`143`)
.. _v2_16:
2.16 (2020-08-21)
-----------------
- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 <https://github.com/simonw/sqlite-utils/issues/134>`__)
- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 <https://github.com/simonw/sqlite-utils/issues/135>`__)
- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (:issue:`134`)
- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (:issue:`135`)
.. _v2_15_1:
2.15.1 (2020-08-12)
-------------------
- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 <https://github.com/simonw/sqlite-utils/issues/133>`__)
- Now available as a ``sdist`` package on PyPI in addition to a wheel. (:issue:`133`)
.. _v2_15:
@ -238,7 +416,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr
-----------------
- New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging <https://www.sqlite.org/wal.html>`__ for a database file - see :ref:`python_api_wal` in the Python API documentation.
- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) <cli_wal>`. (`#132 <https://github.com/simonw/sqlite-utils/issues/132>`__)
- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) <cli_wal>`. (:issue:`132`)
.. _v2_14_1:
@ -252,8 +430,8 @@ This release handles a bug where replacing rows in FTS tables could result in gr
2.14 (2020-08-01)
-----------------
- The :ref:`insert-files command <cli_insert_files>` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 <https://github.com/simonw/sqlite-utils/issues/127>`__)
- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() <python_api_fts>`. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 <https://github.com/simonw/sqlite-utils/issues/130>`__)
- The :ref:`insert-files command <cli_insert_files>` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (:issue:`127`)
- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() <python_api_fts>`. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (:issue:`130`)
- You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts <cli_fts>` CLI command, via the new ``--tokenize`` option.
.. _v2_13:
@ -261,7 +439,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr
2.13 (2020-07-29)
-----------------
- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 <https://github.com/simonw/sqlite-utils/issues/128>`__)
- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (:issue:`128`)
.. _v2_12:
@ -270,11 +448,11 @@ This release handles a bug where replacing rows in FTS tables could result in gr
The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns.
- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 <https://github.com/simonw/sqlite-utils/issues/122>`__)
- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 <https://github.com/simonw/sqlite-utils/issues/123>`__)
- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 <https://github.com/simonw/sqlite-utils/issues/125>`__)
- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 <https://github.com/simonw/sqlite-utils/issues/126>`__)
- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 <https://github.com/simonw/sqlite-utils/issues/124>`__)
- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (:issue:`122`)
- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (:issue:`123`)
- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (:issue:`125`)
- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (:issue:`126`)
- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (:issue:`124`)
.. _v2_11:
@ -289,14 +467,14 @@ The theme of this release is better tools for working with binary data. The new
2.10.1 (2020-06-23)
-------------------
- Added documentation for the ``table.pks`` introspection property. (`#116 <https://github.com/simonw/sqlite-utils/issues/116>`__)
- Added documentation for the ``table.pks`` introspection property. (:issue:`116`)
.. _v2_10:
2.10 (2020-06-12)
-----------------
- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 <https://github.com/simonw/sqlite-utils/issues/115>`__)
- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (:issue:`115`)
.. _v2_9_1:
@ -310,77 +488,77 @@ The theme of this release is better tools for working with binary data. The new
2.9 (2020-05-10)
----------------
- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 <https://github.com/simonw/sqlite-utils/issues/111>`__)
- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (:issue:`111`)
- New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`.
- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 <https://github.com/simonw/sqlite-utils/issues/110>`__)
- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (:issue:`110`)
.. _v2_8:
2.8 (2020-05-03)
----------------
- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 <https://github.com/simonw/sqlite-utils/issues/27>`__)
- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 <https://github.com/simonw/sqlite-utils/issues/107>`__)
- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (:issue:`27`)
- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (:issue:`107`)
.. _v2_7.2:
2.7.2 (2020-05-02)
------------------
- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 <https://github.com/simonw/sqlite-utils/issues/106>`__)
- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (:issue:`106`)
.. _v2_7.1:
2.7.1 (2020-05-01)
------------------
- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 <https://github.com/simonw/sqlite-utils/issues/105>`__)
- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 <https://github.com/simonw/sqlite-utils/issues/104>`__)
- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 <https://github.com/simonw/sqlite-utils/issues/102>`__)
- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (:issue:`105`)
- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (:issue:`104`)
- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (:issue:`102`)
.. _v2_7:
2.7 (2020-04-17)
----------------
- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 <https://github.com/simonw/sqlite-utils/issues/100>`__)
- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (:issue:`100`)
.. _v2_6:
2.6 (2020-04-15)
----------------
- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 <https://github.com/simonw/sqlite-utils/issues/76>`__)
- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (:issue:`76`)
.. _v2_5:
2.5 (2020-04-12)
----------------
- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 <https://github.com/simonw/sqlite-utils/issues/96>`__)
- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 <https://github.com/simonw/sqlite-utils/issues/98>`__)
- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 <https://github.com/simonw/sqlite-utils/issues/97>`__)
- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (:issue:`96`)
- ``table.last_pk`` is now only available for inserts or upserts of a single record. (:issue:`98`)
- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (:issue:`97`)
.. _v2_4_4:
2.4.4 (2020-03-23)
------------------
- Fixed bug where columns with only null values were not correctly created. (`#95 <https://github.com/simonw/sqlite-utils/issues/95>`__)
- Fixed bug where columns with only null values were not correctly created. (:issue:`95`)
.. _v2_4_3:
2.4.3 (2020-03-23)
------------------
- Column type suggestion code is no longer confused by null values. (`#94 <https://github.com/simonw/sqlite-utils/issues/94>`__)
- Column type suggestion code is no longer confused by null values. (:issue:`94`)
.. _v2_4_2:
2.4.2 (2020-03-14)
------------------
- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 <https://github.com/simonw/sqlite-utils/issues/92>`__)
- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (:issue:`92`)
- Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`.
.. _v2_4_1:
@ -388,45 +566,45 @@ The theme of this release is better tools for working with binary data. The new
2.4.1 (2020-03-01)
------------------
- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 <https://github.com/simonw/sqlite-utils/issues/90>`__)
- ``table.enable_fts()`` now works with columns that contain spaces. (:issue:`90`)
.. _v2_4:
2.4 (2020-02-26)
----------------
- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 <https://github.com/simonw/sqlite-utils/issues/88>`__)
- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 <https://github.com/simonw/sqlite-utils/issues/88>`__)
- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 <https://github.com/simonw/sqlite-utils/issues/86>`__)
- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 <https://github.com/simonw/sqlite-utils/issues/87>`__)
- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (:issue:`88`)
- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (:issue:`88`)
- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (:issue:`86`)
- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (:issue:`87`)
.. _v2_3_1:
2.3.1 (2020-02-10)
------------------
``table.create_index()`` now works for columns that contain spaces. (`#85 <https://github.com/simonw/sqlite-utils/issues/85>`__)
``table.create_index()`` now works for columns that contain spaces. (:issue:`85`)
.. _v2_3:
2.3 (2020-02-08)
----------------
``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 <https://github.com/simonw/sqlite-utils/issues/83>`__)
``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (:issue:`83`)
.. _v2_2_1:
2.2.1 (2020-02-06)
------------------
Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 <https://github.com/simonw/sqlite-utils/issues/84>`__).
Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (:issue:`84`).
.. _v2_2:
2.2 (2020-02-01)
----------------
New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 <https://github.com/simonw/sqlite-utils/issues/81>`__).
New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (:issue:`81`).
This replaces the undocumented ``table.detect_column_types()`` method.
@ -442,7 +620,7 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of
2.0.1 (2020-01-05)
------------------
The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 <https://github.com/simonw/sqlite-utils/issues/73>`__).
The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (:issue:`73`).
.. _v2:
@ -464,14 +642,14 @@ For full background on this change, see `issue #66 <https://github.com/simonw/sq
1.12.1 (2019-11-06)
-------------------
- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (`#52 <https://github.com/simonw/sqlite-utils/issues/52>`__)
- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (:issue:`52`)
.. _v1_12:
1.12 (2019-11-04)
-----------------
Python library utilities for deleting records (`#62 <https://github.com/simonw/sqlite-utils/issues/62>`__)
Python library utilities for deleting records (:issue:`62`)
- ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete`
- ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where`
@ -485,14 +663,14 @@ Option to create triggers to automatically keep FTS tables up-to-date with newly
- ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI <cli_fts>`
- ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library <python_api_fts>`
- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 <https://github.com/simonw/sqlite-utils/issues/59>`__)
- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (:issue:`59`)
.. _v1_10:
1.10 (2019-08-23)
-----------------
Ability to introspect and run queries against views (`#54 <https://github.com/simonw/sqlite-utils/issues/54>`__)
Ability to introspect and run queries against views (:issue:`54`)
- ``db.view_names()`` method and and ``db.views`` property
- Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class
@ -505,21 +683,21 @@ See :ref:`python_api_views`.
1.9 (2019-08-04)
----------------
- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 <https://github.com/simonw/sqlite-utils/issues/23>`__)
- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (:issue:`23`)
.. _v1_8:
1.8 (2019-07-28)
----------------
- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 <https://github.com/simonw/sqlite-utils/issues/35>`__)
- ``table.update(pk, values)`` method: :ref:`python_api_update` (:issue:`35`)
.. _v1_7_1:
1.7.1 (2019-07-28)
------------------
- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 <https://github.com/simonw/sqlite-utils/issues/50>`__)
- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (:issue:`50`)
- Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop`
.. _v1_7:
@ -529,8 +707,8 @@ See :ref:`python_api_views`.
Support for lookup tables.
- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 <https://github.com/simonw/sqlite-utils/issues/44>`__)
- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 <https://github.com/simonw/sqlite-utils/issues/46>`__)
- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (:issue:`44`)
- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (:issue:`46`)
- Use `pysqlite3 <https://github.com/coleifer/pysqlite3>`__ if it is available, otherwise use ``sqlite3`` from the standard library
- Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration`
- In-memory databases can now be created using ``db = Database(memory=True)``
@ -540,19 +718,19 @@ Support for lookup tables.
1.6 (2019-07-18)
----------------
- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (`#41 <https://github.com/simonw/sqlite-utils/issues/41>`__)
- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (:issue:`41`)
.. _v1_5:
1.5 (2019-07-14)
----------------
- Support for compound primary keys (`#36 <https://github.com/simonw/sqlite-utils/issues/36>`__)
- Support for compound primary keys (:issue:`36`)
- Configure these using the CLI tool by passing ``--pk`` multiple times
- In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys`
- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 <https://github.com/simonw/sqlite-utils/issues/39>`__)
- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (:issue:`39`)
.. _v1_4_1:
@ -566,14 +744,14 @@ Support for lookup tables.
1.4 (2019-06-30)
----------------
- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs <cli_index_foreign_keys>`) and ``db.index_foreign_keys()`` method (:ref:`docs <python_api_index_foreign_keys>`) (`#33 <https://github.com/simonw/sqlite-utils/issues/33>`__)
- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs <cli_index_foreign_keys>`) and ``db.index_foreign_keys()`` method (:ref:`docs <python_api_index_foreign_keys>`) (:issue:`33`)
.. _v1_3:
1.3 (2019-06-28)
----------------
- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation <python_api_add_foreign_keys>` (`#31 <https://github.com/simonw/sqlite-utils/issues/31>`__)
- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation <python_api_add_foreign_keys>` (:issue:`31`)
.. _v1_2_2:
@ -587,15 +765,15 @@ Support for lookup tables.
1.2.1 (2019-06-20)
------------------
- Check the column exists before attempting to add a foreign key (`#29 <https://github.com/simonw/sqlite-utils/issues/29>`__)
- Check the column exists before attempting to add a foreign key (:issue:`29`)
.. _v1_2:
1.2 (2019-06-12)
----------------
- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by instrospecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 <https://github.com/simonw/sqlite-utils/issues/25>`__)
- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 <https://github.com/simonw/sqlite-utils/issues/24>`__). Documentation: :ref:`Setting defaults and not null constraints (Python API) <python_api_defaults_not_null>`, :ref:`Setting defaults and not null constraints (CLI) <cli_defaults_not_null>`
- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (:issue:`25`)
- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (:issue:`24`). Documentation: :ref:`Setting defaults and not null constraints (Python API) <python_api_defaults_not_null>`, :ref:`Setting defaults and not null constraints (CLI) <cli_defaults_not_null>`
- Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
.. _v1_1:
@ -603,8 +781,8 @@ Support for lookup tables.
1.1 (2019-05-28)
----------------
- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key alread exists (`#21 <https://github.com/simonw/sqlite-utils/issues/21>`__) - documentation: :ref:`Inserting data (Python API) <python_api_bulk_inserts>`, :ref:`Inserting data (CLI) <cli_inserting_data>`
- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 <https://github.com/simonw/sqlite-utils/issues/16>`__) - documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (:issue:`21`) - documentation: :ref:`Inserting data (Python API) <python_api_bulk_inserts>`, :ref:`Inserting data (CLI) <cli_inserting_data>`
- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (:issue:`16`) - documentation: :ref:`Adding columns (Python API) <python_api_add_column>`, :ref:`Adding columns (CLI) <cli_add_column>`
.. _v1_0_1:

628
docs/cli.rst
View file

@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database
.. contents:: :local:
.. _cli_query_json:
.. _cli_query:
Running queries and returning JSON
==================================
Running SQL queries
===================
You can execute a SQL query against a database and get the results back as JSON like this::
The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way::
$ sqlite-utils query dogs.db "select * from dogs"
$ sqlite-utils dogs.db "select * from dogs"
.. _cli_query_json:
Returning JSON
--------------
The default format returned for queries is JSON::
$ sqlite-utils dogs.db "select * from dogs"
[{"id": 1, "age": 4, "name": "Cleo"},
{"id": 2, "age": 2, "name": "Pancakes"}]
This is the default command for ``sqlite-utils``, so you can instead use this::
.. _cli_query_nl:
$ sqlite-utils dogs.db "select * from dogs"
You can pass named parameters to the query using ``-p``::
$ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6
[{":num * :num2": 30}]
Newline-delimited JSON
~~~~~~~~~~~~~~~~~~~~~~
Use ``--nl`` to get back newline-delimited JSON objects::
@ -34,7 +40,12 @@ Use ``--nl`` to get back newline-delimited JSON objects::
{"id": 1, "age": 4, "name": "Cleo"}
{"id": 2, "age": 2, "name": "Pancakes"}
You can use ``--arrays`` to request ararys instead of objects::
.. _cli_query_arrays:
JSON arrays
~~~~~~~~~~~
You can use ``--arrays`` to request arrays instead of objects::
$ sqlite-utils dogs.db "select * from dogs" --arrays
[[1, 4, "Cleo"],
@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python
}
]
.. _cli_query_binary_json:
Binary data in JSON
~~~~~~~~~~~~~~~~~~~
Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this::
$ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool
@ -75,25 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b
}
]
If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows::
$ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'"
[{"rows_affected": 1}]
You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename::
$ sqlite-utils :memory: "select sqlite_version()"
[{"sqlite_version()": "3.29.0"}]
You can load SQLite extension modules using the `--load-extension` option::
$ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib
[{"spatialite_version()": "4.3.0a"}]
.. _cli_json_values:
Nested JSON values
------------------
~~~~~~~~~~~~~~~~~~
If one of your columns contains JSON, by default it will be returned as an escaped string::
@ -124,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column
}
]
.. _cli_attach:
Attaching additional databases
------------------------------
SQLite supports cross-database SQL queries, which can join data from tables in more than one database file.
You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk.
This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database::
sqlite-utils dogs.db --attach books books.db \
'select * from sqlite_master union all select * from books.sqlite_master'
.. _cli_query_csv:
Running queries and returning CSV
=================================
Returning CSV or TSV
--------------------
You can use the ``--csv`` option to return results as CSV::
@ -165,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values::
.. _cli_query_table:
Running queries and outputting a table
======================================
Table-formatted output
----------------------
You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table::
@ -190,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``.
.. _cli_query_raw:
Returning raw data from a query, such as binary content
=======================================================
Returning raw data, such as binary content
------------------------------------------
If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out.
@ -199,6 +187,182 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a
$ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg
.. _cli_query_parameters:
Using named parameters
----------------------
You can pass named parameters to the query using ``-p``::
$ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6
[{":num * :num2": 30}]
These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL.
.. _cli_query_update_insert_delete:
UPDATE, INSERT and DELETE
-------------------------
If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows::
$ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'"
[{"rows_affected": 1}]
SQLite extensions
-----------------
You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`.
::
$ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite
[{"spatialite_version()": "4.3.0a"}]
.. _cli_query_attach:
Attaching additional databases
------------------------------
SQLite supports cross-database SQL queries, which can join data from tables in more than one database file.
You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk.
This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database::
sqlite-utils dogs.db --attach books books.db \
'select * from sqlite_master union all select * from books.sqlite_master'
.. _cli_memory:
Querying data directly using an in-memory database
==================================================
The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database.
You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite.
Without any extra arguments, this command executes SQL against the in-memory database directly::
$ sqlite-utils memory 'select sqlite_version()'
[{"sqlite_version()": "3.35.5"}]
It takes all of the same output formatting options as :ref:`sqlite-utils query <cli_query>`: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``::
$ sqlite-utils memory 'select sqlite_version()' --csv
sqlite_version()
3.35.5
$ sqlite-utils memory 'select sqlite_version()' --table --fmt grid
+--------------------+
| sqlite_version() |
+====================+
| 3.35.5 |
+--------------------+
.. _cli_memory_csv_json:
Running queries directly against CSV or JSON
--------------------------------------------
If you have data in CSV or JSON format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this::
$ sqlite-utils memory data.csv "select * from data"
You can pass multiple files to the command if you want to run joins between data from different files::
$ sqlite-utils memory one.csv two.json "select * from one join two on one.id = two.other_id"
If your data is JSON it should be the same format supported by the :ref:`sqlite-utils insert command <cli_inserting_data>` - so either a single JSON object (treated as a single row) or a list of JSON objects.
CSV data can be comma- or tab- delimited.
The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table::
$ sqlite-utils memory example.csv "select * from t"
To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name::
$ cat example.csv | sqlite-utils memory - "select * from stdin"
Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``::
$ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1
If you are joining across multiple CSV files they must all use the same encoding.
Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``.
.. _cli_memory_explicit:
Explicitly specifying the format
--------------------------------
By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically.
You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example::
$ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two"
Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON.
To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example::
$ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin"
.. _cli_memory_attach:
Joining in-memory data against existing databases using \-\-attach
------------------------------------------------------------------
The :ref:`attach option <cli_query_attach>` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example::
$ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \
"select * from trees.trees where rowid in (select id from stdin)"
Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``.
``select * from trees.trees where ...`` can then query the ``trees`` table in that database.
The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content.
.. _cli_memory_schema_dump_save:
\-\-schema, \-\-dump and \-\-save
---------------------------------
To see the schema that will be created for a file or multiple files, use ``--schema``::
% sqlite-utils memory dogs.csv --schema
CREATE TABLE [dogs] (
[id] INTEGER,
[age] INTEGER,
[name] TEXT
);
CREATE VIEW t1 AS select * from [dogs];
CREATE VIEW t AS select * from [dogs];
You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``::
% sqlite-utils memory dogs.csv --dump
BEGIN TRANSACTION;
CREATE TABLE [dogs] (
[id] INTEGER,
[age] INTEGER,
[name] TEXT
);
INSERT INTO "dogs" VALUES('1','4','Cleo');
INSERT INTO "dogs" VALUES('2','2','Pancakes');
CREATE VIEW t1 AS select * from [dogs];
CREATE VIEW t AS select * from [dogs];
COMMIT;
Passing ``--save other.db`` will instead use that SQL to populate a new database file::
% sqlite-utils memory dogs.csv --save dogs.db
These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`.
.. _cli_rows:
Returning all rows in a table
@ -268,7 +432,7 @@ Use ``--schema`` to include the schema of each table::
[age] INTEGER,
[name] TEXT)
The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available.
The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available.
.. _cli_views:
@ -293,6 +457,33 @@ It takes the same options as the ``tables`` command:
* ``--tsv``
* ``--table``
.. _cli_indexes:
Listing indexes
===============
The ``indexes`` command lists any indexes configured for the database::
$ sqlite-utils indexes covid.db --table
table index_name seqno cid name desc coll key
-------------------------------- ------------------------------------------------------ ------- ----- ----------------- ------ ------ -----
johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_combined_key 0 12 combined_key 0 BINARY 1
johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_country_or_region 0 1 country_or_region 0 BINARY 1
johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_province_or_state 0 2 province_or_state 0 BINARY 1
johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_day 0 0 day 0 BINARY 1
ny_times_us_counties idx_ny_times_us_counties_date 0 0 date 1 BINARY 1
ny_times_us_counties idx_ny_times_us_counties_fips 0 3 fips 0 BINARY 1
ny_times_us_counties idx_ny_times_us_counties_county 0 1 county 0 BINARY 1
ny_times_us_counties idx_ny_times_us_counties_state 0 2 state 0 BINARY 1
It shows indexes across all tables. To see indexes for specific tables, list those after the database::
$ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table
The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``.
The command takes the same format options as the ``tables`` and ``views`` commands.
.. _cli_triggers:
Listing triggers
@ -321,6 +512,24 @@ It defaults to showing triggers for all tables. To see triggers for one or more
The command takes the same format options as the ``tables`` and ``views`` commands.
.. _cli_schema:
Showing the schema
==================
The ``sqlite-utils schema`` command shows the full SQL schema for the database::
$ sqlite-utils schema dogs.db
CREATE TABLE "dogs" (
[id] INTEGER PRIMARY KEY,
[name] TEXT
);
This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments::
$ sqlite-utils schema dogs.db dogs chickens
...
.. _cli_analyze_tables:
Analyzing tables
@ -408,6 +617,21 @@ The ``_analyze_tables_`` table has the following schema::
PRIMARY KEY ([table], [column])
);
The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this::
[
["Del Libertador, Av", 5068],
["Alberdi Juan Bautista Av.", 4612],
["Directorio Av.", 4552],
["Rivadavia, Av", 4532],
["Yerbal", 4512],
["Cosquín", 4472],
["Estado Plurinacional de Bolivia", 4440],
["Gordillo Timoteo", 4424],
["Montiel", 4360],
["Condarco", 4288]
]
.. _cli_inserting_data:
Inserting JSON data
@ -445,6 +669,23 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do
}
]
You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so::
$ sqlite-utils insert dogs.db dogs dogs.json --pk=id
You can skip inserting any records that have a primary key that already exists using ``--ignore``::
$ sqlite-utils insert dogs.db dogs dogs.json --ignore
You can delete all the existing rows in the table before inserting the new records using ``--truncate``::
$ sqlite-utils insert dogs.db dogs dogs.json --truncate
.. _cli_inserting_data_binary:
Inserting binary data
---------------------
You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this::
[
@ -457,17 +698,10 @@ You can insert binary data into a BLOB column by first encoding it using base64
}
]
You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so::
.. _cli_inserting_data_nl_json:
$ sqlite-utils insert dogs.db dogs dogs.json --pk=id
You can skip inserting any records that have a primary key that already exists using ``--ignore``::
$ sqlite-utils insert dogs.db dogs dogs.json --ignore
You can delete all the existing rows in the table before inserting the new records using ``--truncate``::
$ sqlite-utils insert dogs.db dogs dogs.json --truncate
Inserting newline-delimited JSON
--------------------------------
You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette <https://datasette.io/>`__ can export newline-delimited JSON, you can combine the two tools like so::
@ -488,6 +722,49 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite
207368,920 Kirkham St,37.760210314285,-122.47073935813
188702,1501 Evans Ave,37.7422086702947,-122.387293152263
.. _cli_inserting_data_flatten:
Flattening nested JSON objects
------------------------------
``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table.
If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data.
Consider this example document, in a file called ``log.json``::
{
"httpRequest": {
"latency": "0.112114537s",
"requestMethod": "GET",
"requestSize": "534",
"status": 200
},
"insertId": "6111722f000b5b4c4d4071e2",
"labels": {
"service": "datasette-io"
}
}
Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema::
CREATE TABLE [logs] (
[httpRequest] TEXT,
[insertId] TEXT,
[labels] TEXT
);
With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead::
CREATE TABLE [logs] (
[httpRequest_latency] TEXT,
[httpRequest_requestMethod] TEXT,
[httpRequest_requestSize] TEXT,
[httpRequest_status] INTEGER,
[insertId] TEXT,
[labels_service] TEXT
);
.. _cli_insert_csv_tsv:
Inserting CSV or TSV data
@ -507,6 +784,31 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char
A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option.
By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``).
For example, given a ``creatures.csv`` file containing this::
name,age,weight
Cleo,6,45.5
Dori,1,3.5
The following command::
$ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types
Will produce this schema::
$ sqlite-utils schema creatures.db
CREATE TABLE "creatures" (
[name] TEXT,
[age] INTEGER,
[weight] FLOAT
);
You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior::
$ export SQLITE_UTILS_DETECT_TYPES=1
.. _cli_insert_csv_tsv_delimiter:
Alternative delimiters and quote characters
@ -661,6 +963,128 @@ The ``-`` argument indicates data should be read from standard input. The string
When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``.
.. _cli_convert:
Converting data in columns
==========================
The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array.
The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version::
$ sqlite-utils convert content.db articles headline 'value.upper()'
The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column.
The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement::
$ sqlite-utils convert content.db articles headline '
value = str(value)
return value.upper()'
You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters::
$ sqlite-utils convert content.db articles content \
'"\n".join(textwrap.wrap(value, 100))' \
--import=textwrap
The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``::
$ sqlite-utils convert content.db articles headline 'value.upper()' \
--where "headline like '%cat%'"
You can include named parameters in your where clause and populate them using one or more ``--param`` options::
$ sqlite-utils convert content.db articles headline 'value.upper()' \
--where "headline like :like" \
--param like '%cat%'
The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database.
.. _cli_convert_recipes:
sqlite-utils convert recipes
----------------------------
Various built-in recipe functions are available for common operations. These are:
``r.jsonsplit(value, delimiter=',', type=<class 'str'>)``
Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]``
The ``delimiter`` parameter can be used to specify a different delimiter.
The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4.5`` the following::
r.jsonsplit(value, type=float)
Would produce an array like this: ``[1.2, 3.0, 4.5]``
``r.parsedate(value, dayfirst=False, yearfirst=False)``
Parse a date and convert it to ISO date format: ``yyyy-mm-dd``
In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted.
``r.parsedatetime(value, dayfirst=False, yearfirst=False)``
Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS``
These recipes can be used in the code passed to ``sqlite-utils convert`` like this::
$ sqlite-utils convert my.db mytable mycolumn \
'r.jsonsplit(value)'
To use any of the documented parameters, do this::
$ sqlite-utils convert my.db mytable mycolumn \
'r.jsonsplit(value, delimiter=":")'
.. _cli_convert_output:
Saving the result to a different column
---------------------------------------
The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist::
$ sqlite-utils convert content.db articles headline 'value.upper()' \
--output headline_upper
The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5::
$ sqlite-utils convert content.db articles id 'float(value) + 0.5' \
--output id_as_a_float \
--output-type float
You can drop the original column at the end of the operation by adding ``--drop``.
.. _cli_convert_multi:
Converting a column into multiple columns
-----------------------------------------
Sometimes you may wish to convert a single column into multiple derived columns. For example, you may have a ``location`` column containing ``latitude,longitude`` values which you wish to split out into separate ``latitude`` and ``longitude`` columns.
You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary.
For the ``latitude,longitude`` example you would use the following::
$ sqlite-utils convert demo.db places location \
'bits = value.split(",")
return {
"latitude": float(bits[0]),
"longitude": float(bits[1]),
}' --multi
The type of the returned values will be taken into account when creating the new columns. In this example, the resulting database schema will look like this:
.. code-block:: sql
CREATE TABLE [places] (
[location] TEXT,
[latitude] FLOAT,
[longitude] FLOAT
);
The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``.
.. _cli_create_table:
Creating tables
@ -861,45 +1285,44 @@ Here's a more complex example that makes use of these options. It converts `this
--fk-column country_id \
--rename country_long name
After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema:
After running the above, the command ``sqlite-utils schema global.db`` reveals the following schema:
.. code-block:: sql
CREATE TABLE [countries] (
[id] INTEGER PRIMARY KEY,
[country] TEXT,
[name] TEXT
[id] INTEGER PRIMARY KEY,
[country] TEXT,
[name] TEXT
);
CREATE TABLE "power_plants" (
[country_id] INTEGER,
[name] TEXT,
[gppd_idnr] TEXT,
[capacity_mw] TEXT,
[latitude] TEXT,
[longitude] TEXT,
[primary_fuel] TEXT,
[other_fuel1] TEXT,
[other_fuel2] TEXT,
[other_fuel3] TEXT,
[commissioning_year] TEXT,
[owner] TEXT,
[source] TEXT,
[url] TEXT,
[geolocation_source] TEXT,
[wepp_id] TEXT,
[year_of_capacity_data] TEXT,
[generation_gwh_2013] TEXT,
[generation_gwh_2014] TEXT,
[generation_gwh_2015] TEXT,
[generation_gwh_2016] TEXT,
[generation_gwh_2017] TEXT,
[generation_data_source] TEXT,
[estimated_generation_gwh] TEXT,
FOREIGN KEY([country_id]) REFERENCES [countries]([id])
);
CREATE UNIQUE INDEX [idx_countries_country_name]
ON [countries] ([country], [name]);
CREATE TABLE IF NOT EXISTS "power_plants" (
[rowid] INTEGER PRIMARY KEY,
[country_id] INTEGER,
[name] TEXT,
[gppd_idnr] TEXT,
[capacity_mw] TEXT,
[latitude] TEXT,
[longitude] TEXT,
[primary_fuel] TEXT,
[other_fuel1] TEXT,
[other_fuel2] TEXT,
[other_fuel3] TEXT,
[commissioning_year] TEXT,
[owner] TEXT,
[source] TEXT,
[url] TEXT,
[geolocation_source] TEXT,
[wepp_id] TEXT,
[year_of_capacity_data] TEXT,
[generation_gwh_2013] TEXT,
[generation_gwh_2014] TEXT,
[generation_gwh_2015] TEXT,
[generation_gwh_2016] TEXT,
[generation_gwh_2017] TEXT,
[generation_data_source] TEXT,
[estimated_generation_gwh] TEXT,
FOREIGN KEY(country_id) REFERENCES countries(id)
);
.. _cli_create_view:
@ -1033,6 +1456,14 @@ Use the ``--unique`` option to create a unique index.
Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists.
To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this::
$ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3
This will create an index on that table on ``(col1, col2 desc, col3)``.
If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool.
.. _cli_fts:
Configuring full-text search
@ -1181,16 +1612,29 @@ You can disable WAL mode using ``disable-wal``::
Both of these commands accept one or more database files as arguments.
.. _cli_dump:
Dumping the database to SQL
===========================
The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file::
$ sqlite-utils dump mydb.db
BEGIN TRANSACTION;
CREATE TABLE ...
...
COMMIT;
.. _cli_load_extension:
Loading SQLite extensions
=========================
Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``.
Many of these commands have the ability to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``.
This option can be applied multiple times to load multiple extensions.
Since `SpatiaLite <https://www.gaia-gis.it/fossil/libspatialite/index>`__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located::
$ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite
$ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite
[{"spatialite_version()": "4.3.0a"}]

1
docs/codespell-ignore-words.txt Normal file
View file

@ -0,0 +1 @@
doub

7
docs/conf.py
View file

@ -30,7 +30,12 @@ from subprocess import Popen, PIPE
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []
extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"]
autodoc_member_order = "bysource"
extlinks = {
"issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"),
}
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

67
docs/contributing.rst Normal file
View file

@ -0,0 +1,67 @@
.. _contributing:
==============
Contributing
==============
To work on this library locally, first checkout the code. Then create a new virtual environment::
git clone git@github.com:simonw/sqlite-utils
cd sqlite-utils
python3 -mvenv venv
source venv/bin/activate
Or if you are using ``pipenv``::
pipenv shell
Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment.
.. _contributing_tests:
Running the tests
=================
To install the dependencies and test dependencies::
pip install -e '.[test]'
To run the tests::
pytest
.. _contributing_docs:
Building the documentation
==========================
To build the documentation, first install the documentation dependencies::
pip install -e '.[docs]'
Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file::
cd docs
make livehtml
.. _contributing_linting:
Linting and formatting
======================
``sqlite-utils`` uses `Black <https://black.readthedocs.io/>`__ for code formatting, and `flake8 <https://flake8.pycqa.org/>`__ and `mypy <https://mypy.readthedocs.io/>`__ for linting and type checking.
Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project::
black .
To install ``mypy`` and ``flake8`` run the following::
pip install -e '.[flake8,mypy]'
Both commands can then be run in the root of the project like this::
flake8
mypy sqlite_utils
All three of these tools are run by our CI mechanism against every commit and pull request.

0
docs/dogs.db
View file

7
docs/index.rst
View file

@ -13,7 +13,7 @@
.. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
:target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE
*Python utility functions for manipulating SQLite databases*
*CLI tool and Python utility functions for manipulating SQLite databases*
This library and command-line utility helps create SQLite databases from an existing collection of data.
@ -29,8 +29,9 @@ Contents
.. toctree::
:maxdepth: 3
installation
cli
python-api
reference
contributing
changelog
Take a look at `this script <https://github.com/simonw/russian-ira-facebook-ads-datasette/blob/master/fetch_and_build_russian_ads.py>`_ for an example of this library in action.

40
docs/installation.rst Normal file
View file

@ -0,0 +1,40 @@
.. _installation:
==============
Installation
==============
``sqlite-utils`` is tested on Linux, macOS and Windows.
.. _installation_homebrew:
Using Homebrew
==============
The :ref:`sqlite-utils command-line tool <cli>` can be installed on macOS using Homebrew::
brew install sqlite-utils
If you have it installed and want to upgrade to the most recent release, you can run::
brew upgrade sqlite-utils
Then run ``sqlite-utils --version`` to confirm the installed version.
.. _installation_pip:
Using pip
=========
The `sqlite-utils package <https://pypi.org/project/sqlite-utils/>`__ on PyPI includes both the :ref:`sqlite_utils Python library <python_api>` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so::
pip install sqlite-utils
.. _installation_pipx:
Using pipx
==========
`pipx <https://pypi.org/project/pipx/>`__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this::
pipx install sqlite-utils

337
docs/python-api.rst
View file

@ -1,8 +1,8 @@
.. _python_api:
============
Python API
============
=============================
sqlite_utils Python library
=============================
.. contents:: :local:
@ -59,12 +59,11 @@ You can attach an additional database using the ``.attach()`` method, providing
db = Database("first.db")
db.attach("second", "second.db")
# Now you can run queries like this one:
cursor = db.execute("""
print(db.query("""
select * from table_in_first
union all
select * from second.table_in_second
""")
print(cursor.fetchall())
"""))
You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above.
@ -97,27 +96,77 @@ You can also turn on a tracer function temporarily for a block of code using the
This example will print queries only for the duration of the ``with`` block.
.. _python_api_execute:
.. _python_api_executing_queries:
Executing queries
=================
The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered.
The ``Database`` class offers several methods for directly executing SQL queries.
.. _python_api_query:
db.query(sql, params)
---------------------
The ``db.query(sql)`` function executes a SQL query and returns an iterator over Python dictionaries representing the resulting rows:
.. code-block:: python
db = Database(memory=True)
db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
for row in db.query("select * from dogs"):
print(row)
# Outputs:
# {'name': 'Cleo'}
# {'name': 'Pancakes'}
.. _python_api_execute:
db.execute(sql, params)
-----------------------
The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the :ref:`tracer function <python_api_tracing>` if one has been registered.
``db.execute(sql)`` returns a `sqlite3.Cursor <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor>`__ that was used to execute the SQL.
.. code-block:: python
db = Database(memory=True)
db["dogs"].insert({"name": "Cleo"})
db.execute("update dogs set name = 'Cleopaws'")
cursor = db.execute("update dogs set name = 'Cleopaws'")
print(cursor.rowcount)
# Outputs the number of rows affected by the update
# In this case 2
You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped.
Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor>`__.
.. _python_api_parameters:
Passing parameters
------------------
Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query.
This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities.
``?`` parameters in the SQL query can be filled in using a list:
.. code-block:: python
# Using ? and a list:
db.execute("update dogs set name = ?", ["Cleopaws"])
# Or using :name and a dictionary:
db.execute("update dogs set name = :name", {"name": "Cleopaws"})
# This will rename ALL dogs to be called "Cleopaws"
Named parameters using ``:name`` can be filled using a dictionary:
.. code-block:: python
dog = next(db.query(
"select rowid, name from dogs where name = :name",
{"name": "Cleopaws"}
))
# dog is now {'rowid': 1, 'name': 'Cleopaws'}
In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method.
.. _python_api_table:
@ -202,7 +251,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``::
... print(row)
{'id': 1, 'age': 4, 'name': 'Cleo'}
To return custom columns (instead of using ``select *``) pass ``select=``::
The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this::
>>> for row in db["dogs"].rows_where("age > :age", {"age": 3}):
... print(row)
{'id': 1, 'age': 4, 'name': 'Cleo'}
To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``::
>>> db = sqlite_utils.Database("dogs.db")
>>> for row in db["dogs"].rows_where(select='name, age'):
@ -231,6 +286,16 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an
... print(row)
{'id': 1, 'age': 4, 'name': 'Cleo'}
.. _python_api_rows_count_where:
Counting rows
-------------
To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``:
>>> db["dogs"].count_where("age > ?", [1]):
2
.. _python_api_pks_and_rows_where:
Listing rows with their primary keys
@ -291,6 +356,21 @@ If the record does not exist a ``NotFoundError`` will be raised:
except NotFoundError:
print("Dog not found")
.. _python_api_schema:
Showing the schema
==================
The ``db.schema`` property returns the full SQL schema for the database as a string::
>>> db = sqlite_utils.Database("dogs.db")
>>> print(db.schema)
>>> print(db.schema)
CREATE TABLE "dogs" (
[id] INTEGER PRIMARY KEY,
[name] TEXT
);
.. _python_api_creating_tables:
Creating tables
@ -603,7 +683,7 @@ The first argument to ``update()`` is the primary key. This can be a single valu
>>> db["compound_dogs"].update((5, 3), {"name": "Updated"})
The second argument is a dictonary of columns that should be updated, along with their new values.
The second argument is a dictionary of columns that should be updated, along with their new values.
You can cause any missing columns to be added automatically using ``alter=True``::
@ -632,7 +712,7 @@ You can delete all records in a table that match a specific WHERE statement usin
>>> db = sqlite_utils.Database("dogs.db")
>>> # Delete every dog with age less than 3
>>> db["dogs"].delete_where("age < ?", [3]):
>>> db["dogs"].delete_where("age < ?", [3])
Calling ``table.delete_where()`` with no other arguments will delete every row in the table.
@ -647,7 +727,7 @@ For example, given the dogs database you could upsert the record for Cleo like s
.. code-block:: python
db["dogs"].upsert([{
db["dogs"].upsert({
"id": 1,
"name": "Cleo",
"twitter": "cleopaws",
@ -666,6 +746,53 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()`
.. note::
``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
.. _python_api_convert:
Converting data in columns
==========================
The ``table.convert(...)`` method can be used to apply a conversion function to the values in a column, either to update that column or to populate new columns. It is the Python library equivalent of the :ref:`sqlite-utils convert <cli_convert>` command.
This feature works by registering a custom SQLite function that applies a Python transformation, then running a SQL query equivalent to ``UPDATE table SET column = convert_value(column);``
To transform a specific column to uppercase, you would use the following:
.. code-block:: python
db["dogs"].convert("name", lambda value: value.upper())
You can pass a list of columns, in which case the transformation will be applied to each one:
.. code-block:: python
db["dogs"].convert(["name", "twitter"], lambda value: value.upper())
To save the output to of the transformation to a different column, use the ``output=`` parameter:
.. code-block:: python
db["dogs"].convert("name", lambda value: value.upper(), output="name_upper")
This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text.
If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``.
You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column:
.. code-block:: python
table.convert(
"title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True
)
The ``.convert()`` method accepts optional ``where=`` and ``where_args=`` parameters which can be used to apply the conversion to a subset of rows specified by a where clause. Here's how to apply the conversion only to rows with an ``id`` that is higher than 20:
.. code-block:: python
table.convert("title", lambda v: v.upper(), where="id > :id", where_args={"id": 20})
These behave the same as the corresponding parameters to the :ref:`.rows_where() <python_api_rows>` method, so you can use ``?`` placeholders and a list of values instead of ``:named`` placeholders with a dictionary.
.. _python_api_lookup_tables:
Working with lookup tables
@ -835,7 +962,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho
The name of the column
``total_rows``
The total number of rows in the table`
The total number of rows in the table
``num_null``
The number of rows for which this column is null
@ -969,7 +1096,7 @@ Here's an example of this mechanism in action:
])
db["books"].add_foreign_key("author_id", "authors", "id")
The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you ommit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules:
The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules:
- If the column is of format ``author_id``, look for tables called ``author`` or ``authors``
- If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s``
@ -1050,6 +1177,9 @@ The ``table.transform()`` method can do all of these things, by implementing a m
The ``.transform()`` method takes a number of parameters, all of which are optional.
Altering column types
---------------------
To alter the type of a column, use the ``types=`` argument:
.. code-block:: python
@ -1057,6 +1187,11 @@ To alter the type of a column, use the ``types=`` argument:
# Convert the 'age' column to an integer, and 'weight' to a float
table.transform(types={"age": int, "weight": float})
See :ref:`python_api_add_column` for a list of available types.
Renaming columns
----------------
The ``rename=`` parameter can rename columns:
.. code-block:: python
@ -1064,6 +1199,9 @@ The ``rename=`` parameter can rename columns:
# Rename 'age' to 'initial_age':
table.transform(rename={"age": "initial_age"})
Dropping columns
----------------
To drop columns, pass them in the ``drop=`` set:
.. code-block:: python
@ -1071,6 +1209,9 @@ To drop columns, pass them in the ``drop=`` set:
# Drop the 'age' column:
table.transform(drop={"age"})
Changing primary keys
---------------------
To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table.
.. code-block:: python
@ -1078,6 +1219,9 @@ To change the primary key for a table, use ``pk=``. This can be passed a single
# Make `user_id` the new primary key
table.transform(pk="user_id")
Changing not null status
------------------------
You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``:
.. code-block:: python
@ -1095,6 +1239,9 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null
# Make age allow NULL and switch weight to being NOT NULL:
table.transform(not_null={"age": False, "weight": True})
Altering column defaults
------------------------
The ``defaults=`` parameter can be used to set or change the defaults for different columns:
.. code-block:: python
@ -1105,6 +1252,9 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ
# Now remove the default from that column:
table.transform(defaults={"age": None})
Changing column order
---------------------
The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them.
.. code-block:: python
@ -1112,6 +1262,9 @@ The ``column_order=`` parameter can be used to change the order of the columns.
# Change column order
table.transform(column_order=("name", "age", "id")
Dropping foreign key constraints
--------------------------------
You can use ``.transform()`` to remove foreign key constraints from a table.
This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``:
@ -1443,6 +1596,11 @@ If you have loaded an existing table or view, you can use introspection to find
>>> db["PlantType"]
<Table PlantType (id, value)>
.. _python_api_introspection_exists:
.exists()
---------
The ``.exists()`` method can be used to find out if a table exists or not::
>>> db["PlantType"].exists()
@ -1450,6 +1608,11 @@ The ``.exists()`` method can be used to find out if a table exists or not::
>>> db["PlantType2"].exists()
False
.. _python_api_introspection_count:
.count
------
The ``.count`` property shows the current number of rows (``select count(*) from table``)::
>>> db["PlantType"].count
@ -1457,25 +1620,60 @@ The ``.count`` property shows the current number of rows (``select count(*) from
>>> db["Street_Tree_List"].count
189144
This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property.
This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property.
The ``.columns`` property shows the columns in the table or view::
.. _python_api_introspection_columns:
.columns
--------
The ``.columns`` property shows the columns in the table or view. It returns a list of ``Column(cid, name, type, notnull, default_value, is_pk)`` named tuples.
::
>>> db["PlantType"].columns
[Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1),
Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)]
The ``.columns_dict`` property returns a dictionary version of this with just the names and types::
.. _python_api_introspection_columns_dict:
.columns_dict
-------------
The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types::
>>> db["PlantType"].columns_dict
{'id': <class 'int'>, 'value': <class 'str'>}
.. _python_api_introspection_pks:
.pks
----
The ``.pks`` property returns a list of strings naming the primary key columns for the table::
>>> db["PlantType"].pks
['id']
The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views.
If a table has no primary keys but is a `rowid table <https://www.sqlite.org/rowidtable.html>`__, this property will return ``['rowid']``.
.. _python_api_introspection_use_rowid:
.use_rowid
----------
Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly defined primary keys must use that ``rowid`` as the primary key for identifying individual rows. The ``.use_rowid`` property checks to see if a table needs to use the ``rowid`` in this way - it returns ``True`` if the table has no explicitly defined primary keys and ``False`` otherwise.
>>> db["PlantType"].use_rowid
False
.. _python_api_introspection_foreign_keys:
.foreign_keys
-------------
The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views.
::
@ -1487,6 +1685,11 @@ The ``.foreign_keys`` property shows if the table has any foreign key relationsh
ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'),
ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')]
.. _python_api_introspection_schema:
.schema
-------
The ``.schema`` property outputs the table's schema as a SQL string::
>>> print(db["Street_Tree_List"].schema)
@ -1517,7 +1720,12 @@ The ``.schema`` property outputs the table's schema as a SQL string::
FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id),
FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id))
The ``.indexes`` property shows you all indexes created for a table. It is not available on views.
.. _python_api_introspection_indexes:
.indexes
--------
The ``.indexes`` property returns all indexes created for a table, as a list of ``Index(seq, name, unique, origin, partial, columns)`` named tuples. It is not available on views.
::
@ -1529,7 +1737,39 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a
Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']),
Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])]
The ``.triggers`` property lists database triggers. It can be used on both database and table objects.
.. _python_api_introspection_xindexes:
.xindexes
---------
The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() <https://sqlite.org/pragma.html#pragma_index_xinfo>`__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples.
::
>>> db["ny_times_us_counties"].xindexes
[
XIndex(
name='idx_ny_times_us_counties_date',
columns=[
XIndexColumn(seqno=0, cid=0, name='date', desc=1, coll='BINARY', key=1),
XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0)
]
),
XIndex(
name='idx_ny_times_us_counties_fips',
columns=[
XIndexColumn(seqno=0, cid=3, name='fips', desc=0, coll='BINARY', key=1),
XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0)
]
)
]
.. _python_api_introspection_triggers:
.triggers
---------
The ``.triggers`` property lists database triggers. It can be used on both database and table objects. It returns a list of ``Trigger(name, table, sql)`` named tuples.
::
@ -1540,6 +1780,11 @@ The ``.triggers`` property lists database triggers. It can be used on both datab
>>> db.triggers
... similar output to db["authors"].triggers
.. _python_api_introspection_triggers_dict:
.triggers_dict
--------------
The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions.
::
@ -1558,6 +1803,11 @@ The same property exists on the database, and will return all triggers across al
'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...',
'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'}
.. _python_api_introspection_detect_fts:
.detect_fts()
-------------
The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``.
::
@ -1565,12 +1815,22 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one
>>> db["authors"].detect_fts()
"authors_fts"
.. _python_api_introspection_virtual_table_using:
.virtual_table_using
--------------------
The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example::
>>> db["authors"].enable_fts(["name"])
>>> db["authors_fts"].virtual_table_using
"FTS5"
.. _python_api_introspection_has_counts_triggers:
.has_counts_triggers
--------------------
The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`.
::
@ -1848,6 +2108,8 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y
db.reset_counts()
.. _python_api_create_index:
Creating indexes
================
@ -1866,6 +2128,17 @@ By default the index will be named ``idx_{table-name}_{columns}`` - if you want
index_name="good_dogs_by_age"
)
To create an index in descending order for a column, wrap the column name in ``db.DescIndex()`` like this:
.. code-block:: python
from sqlite_utils.db import DescIndex
db["dogs"].create_index(
["is_good_dog", DescIndex("age")],
index_name="good_dogs_by_age"
)
You can create a unique index by passing ``unique=True``:
.. code-block:: python
@ -2050,12 +2323,24 @@ If you want to deliberately replace the registered function with a new implement
def reverse_string(s):
return s[::-1]
Exceptions that occur inside a user-defined function default to returning the following error::
Unexpected error: user-defined function raised exception
You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom functions are executed:
.. code-block:: python
from sqlite_utils.utils import sqlite3
sqlite3.enable_callback_tracebacks(True)
.. _python_api_quote:
Quoting strings for use in SQL
==============================
In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`.
In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.query()``, as described in :ref:`python_api_parameters`.
If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so:

70
docs/reference.rst Normal file
View file

@ -0,0 +1,70 @@
.. _reference:
===============
API Reference
===============
.. contents:: :local:
.. _reference_db_database:
sqlite_utils.db.Database
========================
.. autoclass:: sqlite_utils.db.Database
:members:
:undoc-members:
:special-members: __getitem__
:exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys
.. _reference_db_queryable:
sqlite_utils.db.Queryable
=========================
:ref:`Table <reference_db_table>` and :ref:`View <reference_db_view>` are both subclasses of ``Queryable``, providing access to the following methods:
.. autoclass:: sqlite_utils.db.Queryable
:members:
:undoc-members:
:exclude-members: execute_count
.. _reference_db_table:
sqlite_utils.db.Table
=====================
.. autoclass:: sqlite_utils.db.Table
:members:
:undoc-members:
:show-inheritance:
:exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns
.. _reference_db_view:
sqlite_utils.db.View
====================
.. autoclass:: sqlite_utils.db.View
:members:
:undoc-members:
:show-inheritance:
.. _reference_db_other:
Other
=====
.. _reference_db_other_column:
sqlite_utils.db.Column
----------------------
.. autoclass:: sqlite_utils.db.Column
.. _reference_db_other_column_details:
sqlite_utils.db.ColumnDetails
-----------------------------
.. autoclass:: sqlite_utils.db.ColumnDetails

1053
docs/tutorial.ipynb Normal file
View file

File diff suppressed because it is too large Load diff

3
setup.cfg Normal file
View file

@ -0,0 +1,3 @@
[flake8]
max-line-length = 160
extend-ignore = E203 # for Black

14
setup.py
View file

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
import io
import os
VERSION = "3.6"
VERSION = "3.15.1"
def get_long_description():
@ -22,11 +22,19 @@ setup(
version=VERSION,
license="Apache License, Version 2.0",
packages=find_packages(exclude=["tests", "tests.*"]),
install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"],
install_requires=[
"sqlite-fts4",
"click",
"click-default-group",
"tabulate",
"dateutils",
],
setup_requires=["pytest-runner"],
extras_require={
"test": ["pytest", "black", "hypothesis"],
"docs": ["sphinx_rtd_theme", "sphinx-autobuild"],
"docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
"mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"],
"flake8": ["flake8"],
},
entry_points="""
[console_scripts]

581
sqlite_utils/cli.py
View file

@ -1,12 +1,14 @@
import base64
import click
from click_default_group import DefaultGroup
from click_default_group import DefaultGroup # type: ignore
from datetime import datetime
import hashlib
import pathlib
import sqlite_utils
from sqlite_utils.db import AlterError
from sqlite_utils.db import AlterError, BadMultiValues, DescIndex
from sqlite_utils import recipes
import textwrap
import inspect
import io
import itertools
import json
@ -14,7 +16,18 @@ import os
import sys
import csv as csv_std
import tabulate
from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values
from .utils import (
file_progress,
find_spatialite,
sqlite3,
decode_base64_values,
progressbar,
rows_from_file,
Format,
TypeTracker,
)
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB")
@ -31,7 +44,7 @@ It's often worth trying: --encoding=latin-1
""".strip()
# Increase CSV field size limit to maximim possible
# Increase CSV field size limit to maximum possible
# https://stackoverflow.com/a/15063941
field_size_limit = sys.maxsize
@ -89,7 +102,12 @@ def load_extension_option(fn):
)(fn)
@click.group(cls=DefaultGroup, default="query", default_if_no_args=True)
@click.group(
cls=DefaultGroup,
default="query",
default_if_no_args=True,
context_settings=CONTEXT_SETTINGS,
)
@click.version_option()
def cli():
"Commands for interacting with a SQLite database"
@ -245,17 +263,6 @@ def views(
)
@cli.command()
@click.argument(
"path",
type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
def vacuum(path):
"""Run VACUUM against the database"""
sqlite_utils.Database(path).vacuum()
@cli.command()
@click.argument(
"path",
@ -308,6 +315,21 @@ def vacuum(path):
sqlite_utils.Database(path).vacuum()
@cli.command()
@click.argument(
"path",
type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@load_extension_option
def dump(path, load_extension):
"""Output a SQL dump of the schema and full contents of the database"""
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
for line in db.conn.iterdump():
click.echo(line)
@cli.command(name="add-column")
@click.argument(
"path",
@ -450,11 +472,21 @@ def index_foreign_keys(path, load_extension):
)
@load_extension_option
def create_index(path, table, column, name, unique, if_not_exists, load_extension):
"Add an index to the specified table covering the specified columns"
"""
Add an index to the specified table covering the specified columns.
Use "sqlite-utils create-index mydb -- -column" to specify descending
order for a column.
"""
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
# Treat -prefix as descending for columns
columns = []
for col in column:
if col.startswith("-"):
col = DescIndex(col[1:])
columns.append(col)
db[table].create_index(
column, index_name=name, unique=unique, if_not_exists=if_not_exists
columns, index_name=name, unique=unique, if_not_exists=if_not_exists
)
@ -611,6 +643,7 @@ def insert_upsert_options(fn):
"--pk", help="Columns to use as the primary key, e.g. id", multiple=True
),
click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"),
click.option("-c", "--csv", is_flag=True, help="Expect CSV"),
click.option("--tsv", is_flag=True, help="Expect TSV"),
click.option("--delimiter", help="Delimiter to use for CSV files"),
@ -644,6 +677,13 @@ def insert_upsert_options(fn):
"--encoding",
help="Character encoding for input, defaults to utf-8",
),
click.option(
"-d",
"--detect-types",
is_flag=True,
envvar="SQLITE_UTILS_DETECT_TYPES",
help="Detect types for columns in CSV/TSV data",
),
load_extension_option,
click.option("--silent", is_flag=True, help="Do not show progress bar"),
)
@ -658,6 +698,7 @@ def insert_upsert_implementation(
json_file,
pk,
nl,
flatten,
csv,
tsv,
delimiter,
@ -673,6 +714,7 @@ def insert_upsert_implementation(
not_null=None,
default=None,
encoding=None,
detect_types=None,
load_extension=None,
silent=False,
):
@ -682,13 +724,16 @@ def insert_upsert_implementation(
csv = True
if (nl + csv + tsv) >= 2:
raise click.ClickException("Use just one of --nl, --csv or --tsv")
if (csv or tsv) and flatten:
raise click.ClickException("--flatten cannot be used with --csv or --tsv")
if encoding and not (csv or tsv):
raise click.ClickException("--encoding must be used with --csv or --tsv")
encoding = encoding or "utf-8"
buffered = io.BufferedReader(json_file, buffer_size=4096)
decoded = io.TextIOWrapper(buffered, encoding=encoding)
if pk and len(pk) == 1:
pk = pk[0]
encoding = encoding or "utf-8-sig"
buffered = io.BufferedReader(json_file, buffer_size=4096)
decoded = io.TextIOWrapper(buffered, encoding=encoding)
tracker = None
if csv or tsv:
if sniff:
# Read first 2048 bytes and use that to detect
@ -710,6 +755,9 @@ def insert_upsert_implementation(
else:
headers = first_row
docs = (dict(zip(headers, row)) for row in reader)
if detect_types:
tracker = TypeTracker()
docs = tracker.wrap(docs)
else:
try:
if nl:
@ -722,6 +770,8 @@ def insert_upsert_implementation(
raise click.ClickException(
"Invalid JSON - use --csv for CSV or --tsv for TSV files"
)
if flatten:
docs = (dict(_flatten(doc)) for doc in docs)
extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate}
if not_null:
@ -732,9 +782,52 @@ def insert_upsert_implementation(
extra_kwargs["upsert"] = upsert
# Apply {"$base64": true, ...} decoding, if needed
docs = (decode_base64_values(doc) for doc in docs)
db[table].insert_all(
docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
)
try:
db[table].insert_all(
docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
)
except Exception as e:
if (
isinstance(e, sqlite3.OperationalError)
and e.args
and "has no column named" in e.args[0]
):
raise click.ClickException(
"{}\n\nTry using --alter to add additional columns".format(e.args[0])
)
# If we can find sql= and parameters= arguments, show those
variables = _find_variables(e.__traceback__, ["sql", "parameters"])
if "sql" in variables and "parameters" in variables:
raise click.ClickException(
"{}\n\nsql = {}\nparameters = {}".format(
str(e), variables["sql"], variables["parameters"]
)
)
else:
raise
if tracker is not None:
db[table].transform(types=tracker.types)
def _flatten(d):
for key, value in d.items():
if isinstance(value, dict):
for key2, value2 in _flatten(value):
yield key + "_" + key2, value2
else:
yield key, value
def _find_variables(tb, vars):
to_find = list(vars)
found = {}
for var in to_find:
if var in tb.tb_frame.f_locals:
vars.remove(var)
found[var] = tb.tb_frame.f_locals[var]
if vars and tb.tb_next:
found.update(_find_variables(tb.tb_next, vars))
return found
@cli.command()
@ -760,6 +853,7 @@ def insert(
json_file,
pk,
nl,
flatten,
csv,
tsv,
delimiter,
@ -769,6 +863,7 @@ def insert(
batch_size,
alter,
encoding,
detect_types,
load_extension,
silent,
ignore,
@ -790,6 +885,7 @@ def insert(
json_file,
pk,
nl,
flatten,
csv,
tsv,
delimiter,
@ -803,6 +899,7 @@ def insert(
replace=replace,
truncate=truncate,
encoding=encoding,
detect_types=detect_types,
load_extension=load_extension,
silent=silent,
not_null=not_null,
@ -820,6 +917,7 @@ def upsert(
json_file,
pk,
nl,
flatten,
csv,
tsv,
batch_size,
@ -831,6 +929,7 @@ def upsert(
not_null,
default,
encoding,
detect_types,
load_extension,
silent,
):
@ -846,6 +945,7 @@ def upsert(
json_file,
pk,
nl,
flatten,
csv,
tsv,
delimiter,
@ -905,7 +1005,17 @@ def upsert(
def create_table(
path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension
):
"Add an index to the specified table covering the specified columns"
"""
Add a table with the specified columns. Columns should be specified using
name, type pairs, for example:
\b
sqlite-utils create-table my.db people \\
id integer \\
name text \\
height float \\
photo blob --pk id
"""
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
if len(columns) % 2 == 1:
@ -1060,8 +1170,168 @@ def query(
db.attach(alias, attach_path)
_load_extensions(db, load_extension)
db.register_fts4_bm25()
_execute_query(
db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
)
@cli.command()
@click.argument(
"paths",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=True),
required=False,
nargs=-1,
)
@click.argument("sql")
@click.option(
"--attach",
type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)),
multiple=True,
help="Additional databases to attach - specify alias and filepath",
)
@output_options
@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row")
@click.option(
"-p",
"--param",
multiple=True,
type=(str, str),
help="Named :parameters for SQL query",
)
@click.option(
"--encoding",
help="Character encoding for CSV input, defaults to utf-8",
)
@click.option(
"-n",
"--no-detect-types",
is_flag=True,
help="Treat all CSV/TSV columns as TEXT",
)
@click.option("--schema", is_flag=True, help="Show SQL schema for in-memory database")
@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database")
@click.option(
"--save",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
help="Save in-memory database to this file",
)
@load_extension_option
def memory(
paths,
sql,
attach,
nl,
arrays,
csv,
tsv,
no_headers,
table,
fmt,
json_cols,
raw,
param,
encoding,
no_detect_types,
schema,
dump,
save,
load_extension,
):
"""Execute SQL query against an in-memory database, optionally populated by imported data
To import data from CSV, TSV or JSON files pass them on the command-line:
\b
sqlite-utils memory one.csv two.json \\
"select * from one join two on one.two_id = two.id"
For data piped into the tool from standard input, use "-" or "stdin":
\b
cat animals.csv | sqlite-utils memory - \\
"select * from stdin where species = 'dog'"
The format of the data will be automatically detected. You can specify the format
explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) - for example:
\b
cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\
"select * from stdin where place_id in (select id from places)"
Use --schema to view the SQL schema of any imported files:
\b
sqlite-utils memory animals.csv --schema
"""
db = sqlite_utils.Database(memory=True)
# If --dump or --save used but no paths detected, assume SQL query is a path:
if (dump or save or schema) and not paths:
paths = [sql]
sql = None
for i, path in enumerate(paths):
# Path may have a :format suffix
if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__:
path, suffix = path.rsplit(":", 1)
format = Format[suffix.upper()]
else:
format = None
if path in ("-", "stdin"):
csv_fp = sys.stdin.buffer
csv_table = "stdin"
else:
csv_path = pathlib.Path(path)
csv_table = csv_path.stem
csv_fp = csv_path.open("rb")
rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding)
tracker = None
if format_used in (Format.CSV, Format.TSV) and not no_detect_types:
tracker = TypeTracker()
rows = tracker.wrap(rows)
db[csv_table].insert_all(rows, alter=True)
if tracker is not None:
db[csv_table].transform(types=tracker.types)
# Add convenient t / t1 / t2 views
view_names = ["t{}".format(i + 1)]
if i == 0:
view_names.append("t")
for view_name in view_names:
if not db[view_name].exists():
db.create_view(view_name, "select * from [{}]".format(csv_table))
if dump:
for line in db.conn.iterdump():
click.echo(line)
return
if schema:
click.echo(db.schema)
return
if save:
db2 = sqlite_utils.Database(save)
for line in db.conn.iterdump():
db2.execute(line)
return
for alias, attach_path in attach:
db.attach(alias, attach_path)
_load_extensions(db, load_extension)
db.register_fts4_bm25()
_execute_query(
db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
)
def _execute_query(
db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols
):
with db.conn:
cursor = db.execute(sql, dict(param))
try:
cursor = db.execute(sql, dict(param))
except sqlite3.OperationalError as e:
raise click.ClickException(str(e))
if cursor.description is None:
# This was an update/insert
headers = ["rows_affected"]
@ -1260,6 +1530,90 @@ def triggers(
)
@cli.command()
@click.argument(
"path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@click.argument("tables", nargs=-1)
@click.option("--aux", is_flag=True, help="Include auxiliary columns")
@output_options
@load_extension_option
@click.pass_context
def indexes(
ctx,
path,
tables,
aux,
nl,
arrays,
csv,
tsv,
no_headers,
table,
fmt,
json_cols,
load_extension,
):
"Show indexes for this database"
sql = """
select
sqlite_master.name as "table",
indexes.name as index_name,
xinfo.*
from sqlite_master
join pragma_index_list(sqlite_master.name) indexes
join pragma_index_xinfo(index_name) xinfo
where
sqlite_master.type = 'table'
"""
if tables:
quote = sqlite_utils.Database(memory=True).quote
sql += " and sqlite_master.name in ({})".format(
", ".join(quote(table) for table in tables)
)
if not aux:
sql += " and xinfo.key = 1"
ctx.invoke(
query,
path=path,
sql=sql,
nl=nl,
arrays=arrays,
csv=csv,
tsv=tsv,
no_headers=no_headers,
table=table,
fmt=fmt,
json_cols=json_cols,
load_extension=load_extension,
)
@cli.command()
@click.argument(
"path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@click.argument("tables", nargs=-1, required=False)
@load_extension_option
def schema(
path,
tables,
load_extension,
):
"Show full schema for this database or for specified tables"
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
if tables:
for table in tables:
click.echo(db[table].schema)
else:
click.echo(db.schema)
@cli.command()
@click.argument(
"path",
@ -1269,9 +1623,12 @@ def triggers(
@click.argument("table")
@click.option(
"--type",
type=(str, str),
type=(
str,
click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False),
),
multiple=True,
help="Change column type to X",
help="Change column type to INTEGER, TEXT, FLOAT or BLOB",
)
@click.option("--drop", type=str, multiple=True, help="Drop this column")
@click.option(
@ -1432,9 +1789,20 @@ def extract(
@click.option("--replace", is_flag=True, help="Replace files with matching primary key")
@click.option("--upsert", is_flag=True, help="Upsert files with matching primary key")
@click.option("--name", type=str, help="File name to use")
@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
@load_extension_option
def insert_files(
path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension
path,
table,
file_or_dir,
column,
pk,
alter,
replace,
upsert,
name,
silent,
load_extension,
):
"""
Insert one or more files using BLOB columns in the specified table
@ -1471,7 +1839,7 @@ def insert_files(
# Load all paths so we can show a progress bar
paths_and_relative_paths = list(yield_paths_and_relative_paths())
with click.progressbar(paths_and_relative_paths) as bar:
with progressbar(paths_and_relative_paths, silent=silent) as bar:
def to_insert():
for path, relative_path in bar:
@ -1593,6 +1961,159 @@ def analyze_tables(
click.echo(details)
def _generate_convert_help():
help = textwrap.dedent(
"""
Convert columns using Python code you supply. For example:
\b
$ sqlite-utils convert my.db mytable mycolumn \\
'"\\n".join(textwrap.wrap(value, 10))' \\
--import=textwrap
"value" is a variable with the column value to be converted.
The following common operations are available as recipe functions:
"""
).strip()
recipe_names = [
n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")
]
for name in recipe_names:
fn = getattr(recipes, name)
help += "\n\nr.{}{}\n\n {}".format(
name, str(inspect.signature(fn)), fn.__doc__
)
help += "\n\n"
help += textwrap.dedent(
"""
You can use these recipes like so:
\b
$ sqlite-utils convert my.db mytable mycolumn \\
'r.jsonsplit(value, delimiter=":")'
"""
).strip()
return help
@cli.command(help=_generate_convert_help())
@click.argument(
"db_path",
type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@click.argument("table", type=str)
@click.argument("columns", type=str, nargs=-1, required=True)
@click.argument("code", type=str)
@click.option(
"--import", "imports", type=str, multiple=True, help="Python modules to import"
)
@click.option(
"--dry-run", is_flag=True, help="Show results of running this against first 10 rows"
)
@click.option(
"--multi", is_flag=True, help="Populate columns for keys in returned dictionary"
)
@click.option("--where", help="Optional where clause")
@click.option(
"-p",
"--param",
multiple=True,
type=(str, str),
help="Named :parameters for where clause",
)
@click.option("--output", help="Optional separate column to populate with the output")
@click.option(
"--output-type",
help="Column type to use for the output column",
default="text",
type=click.Choice(["integer", "float", "blob", "text"]),
)
@click.option("--drop", is_flag=True, help="Drop original column afterwards")
@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
def convert(
db_path,
table,
columns,
code,
imports,
dry_run,
multi,
where,
param,
output,
output_type,
drop,
silent,
):
sqlite3.enable_callback_tracebacks(True)
db = sqlite_utils.Database(db_path)
if output is not None and len(columns) > 1:
raise click.ClickException("Cannot use --output with more than one column")
if multi and len(columns) > 1:
raise click.ClickException("Cannot use --multi with more than one column")
if drop and not (output or multi):
raise click.ClickException("--drop can only be used with --output or --multi")
# If single line and no 'return', add the return
if "\n" not in code and not code.strip().startswith("return "):
code = "return {}".format(code)
where_args = dict(param) if param else []
# Compile the code into a function body called fn(value)
new_code = ["def fn(value):"]
for line in code.split("\n"):
new_code.append(" {}".format(line))
code_o = compile("\n".join(new_code), "<string>", "exec")
locals = {}
globals = {"r": recipes, "recipes": recipes}
for import_ in imports:
globals[import_] = __import__(import_)
exec(code_o, globals, locals)
fn = locals["fn"]
if dry_run:
# Pull first 20 values for first column and preview them
db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v)
sql = """
select
[{column}] as value,
preview_transform([{column}]) as preview
from [{table}]{where} limit 10
""".format(
column=columns[0],
table=table,
where=" where {}".format(where) if where is not None else "",
)
for row in db.conn.execute(sql, where_args).fetchall():
click.echo(str(row[0]))
click.echo(" --- becomes:")
click.echo(str(row[1]))
click.echo()
count = db[table].count_where(
where=where,
where_args=where_args,
)
click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s"))
else:
try:
db[table].convert(
columns,
fn,
where=where,
where_args=where_args,
output=output,
output_type=output_type,
drop=drop,
multi=multi,
show_progress=not silent,
)
except BadMultiValues as e:
raise click.ClickException(
"When using --multi code must return a Python dictionary - returned: {}".format(
repr(e.values)
)
)
def _render_common(title, values):
if values is None:
return ""

1023
sqlite_utils/db.py
View file

File diff suppressed because it is too large Load diff

19
sqlite_utils/recipes.py Normal file
View file

@ -0,0 +1,19 @@
from dateutil import parser
import json
def parsedate(value, dayfirst=False, yearfirst=False):
"Parse a date and convert it to ISO date format: yyyy-mm-dd"
return (
parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat()
)
def parsedatetime(value, dayfirst=False, yearfirst=False):
"Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS"
return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
def jsonsplit(value, delimiter=",", type=str):
'Convert a string like a,b,c into a JSON array ["a", "b", "c"]'
return json.dumps([type(s.strip()) for s in value.split(delimiter)])

179
sqlite_utils/utils.py
View file

@ -1,16 +1,22 @@
import base64
import click
import contextlib
import csv
import enum
import io
import json
import os
from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type
import click
try:
import pysqlite3 as sqlite3
import pysqlite3.dbapi2
import pysqlite3 as sqlite3 # type: ignore
import pysqlite3.dbapi2 # type: ignore
OperationalError = pysqlite3.dbapi2.OperationalError
except ImportError:
import sqlite3
# https://github.com/python/mypy/issues/1153#issuecomment-253842414
import sqlite3 # type: ignore
OperationalError = sqlite3.OperationalError
@ -25,8 +31,11 @@ def suggest_column_types(records):
for record in records:
for key, value in record.items():
all_column_types.setdefault(key, set()).add(type(value))
column_types = {}
return types_for_column_types(all_column_types)
def types_for_column_types(all_column_types):
column_types = {}
for key, types in all_column_types.items():
# Ignore null values if at least one other type present:
if len(types) > 1:
@ -105,9 +114,167 @@ class UpdateWrapper:
@contextlib.contextmanager
def file_progress(file, silent=False, **kwargs):
if silent or file.fileno() == 0: # 0 = stdin
if silent:
yield file
return
# file.fileno() throws an exception in our test suite
try:
fileno = file.fileno()
except io.UnsupportedOperation:
yield file
return
if fileno == 0: # 0 means stdin
yield file
else:
file_length = os.path.getsize(file.name)
with click.progressbar(length=file_length, **kwargs) as bar:
yield UpdateWrapper(file, bar.update)
class Format(enum.Enum):
CSV = 1
TSV = 2
JSON = 3
NL = 4
class RowsFromFileError(Exception):
pass
class RowsFromFileBadJSON(RowsFromFileError):
pass
def rows_from_file(
fp: BinaryIO,
format: Optional[Format] = None,
dialect: Optional[Type[csv.Dialect]] = None,
encoding: Optional[str] = None,
) -> Tuple[Iterable[dict], Format]:
if format == Format.JSON:
decoded = json.load(fp)
if isinstance(decoded, dict):
decoded = [decoded]
if not isinstance(decoded, list):
raise RowsFromFileBadJSON("JSON must be a list or a dictionary")
return decoded, Format.JSON
elif format == Format.NL:
return (json.loads(line) for line in fp if line.strip()), Format.NL
elif format == Format.CSV:
use_encoding: str = encoding or "utf-8-sig"
decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding)
if dialect is not None:
reader = csv.DictReader(decoded_fp, dialect=dialect)
else:
reader = csv.DictReader(decoded_fp)
return reader, Format.CSV
elif format == Format.TSV:
return (
rows_from_file(
fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
)[0],
Format.TSV,
)
elif format is None:
# Detect the format, then call this recursively
buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
first_bytes = buffered.peek(2048).strip()
if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"):
# TODO: Detect newline-JSON
return rows_from_file(buffered, format=Format.JSON)
else:
dialect = csv.Sniffer().sniff(
first_bytes.decode(encoding or "utf-8-sig", "ignore")
)
return rows_from_file(
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
)
else:
raise RowsFromFileError("Bad format")
class TypeTracker:
def __init__(self):
self.trackers = {}
def wrap(self, iterator):
for row in iterator:
for key, value in row.items():
tracker = self.trackers.setdefault(key, ValueTracker())
tracker.evaluate(value)
yield row
@property
def types(self):
return {key: tracker.guessed_type for key, tracker in self.trackers.items()}
class ValueTracker:
def __init__(self):
self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()}
@classmethod
def get_tests(cls):
return [
key.split("test_")[-1]
for key in cls.__dict__.keys()
if key.startswith("test_")
]
def test_integer(self, value):
try:
int(value)
return True
except (ValueError, TypeError):
return False
def test_float(self, value):
try:
float(value)
return True
except (ValueError, TypeError):
return False
def __repr__(self):
return self.guessed_type + ": possibilities = " + repr(self.couldbe)
@property
def guessed_type(self):
options = set(self.couldbe.keys())
# Return based on precedence
for key in self.get_tests():
if key in options:
return key
return "text"
def evaluate(self, value):
if not value or not self.couldbe:
return
not_these = []
for name, test in self.couldbe.items():
if not test(value):
not_these.append(name)
for key in not_these:
del self.couldbe[key]
class NullProgressBar:
def __init__(self, *args):
self.args = args
def __iter__(self):
yield from self.args[0]
def update(self, value):
pass
@contextlib.contextmanager
def progressbar(*args, **kwargs):
silent = kwargs.pop("silent")
if silent:
yield NullProgressBar(*args)
else:
with click.progressbar(*args, **kwargs) as bar:
yield bar

5
tests/test_analyze_tables.py
View file

@ -1,10 +1,8 @@
from sqlite_utils.db import Database, ForeignKey, ColumnDetails
from sqlite_utils.db import Database, ColumnDetails
from sqlite_utils import cli
from sqlite_utils.utils import OperationalError
from click.testing import CliRunner
import pytest
import sqlite3
import textwrap
@pytest.fixture
@ -132,6 +130,7 @@ def test_analyze_table_save(db_to_analyze_path):
result = CliRunner().invoke(
cli.cli, ["analyze-tables", db_to_analyze_path, "--save"]
)
assert result.exit_code == 0
rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows)
assert rows == [
{

536
tests/test_cli.py
View file

@ -1,6 +1,7 @@
from sqlite_utils import cli, Database
from sqlite_utils.db import Index, ForeignKey
from click.testing import CliRunner
from unittest import mock
import json
import os
import pytest
@ -24,6 +25,22 @@ def db_path(tmpdir):
return path
@pytest.mark.parametrize(
"options",
(
["-h"],
["--help"],
["insert", "-h"],
["insert", "--help"],
),
)
def test_help(options):
result = CliRunner().invoke(cli.cli, options)
assert result.exit_code == 0
assert result.output.startswith("Usage: ")
assert "-h, --help" in result.output
def test_tables(db_path):
result = CliRunner().invoke(cli.cli, ["tables", db_path])
assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip()
@ -208,6 +225,17 @@ def test_create_index(db_path):
)
def test_create_index_desc(db_path):
db = Database(db_path)
assert [] == db["Gosh"].indexes
result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"])
assert result.exit_code == 0
assert (
db.execute("select sql from sqlite_master where type='index'").fetchone()[0]
== "CREATE INDEX [idx_Gosh_c1]\n ON [Gosh] ([c1] desc)"
)
@pytest.mark.parametrize(
"col_name,col_type,expected_schema",
(
@ -354,6 +382,21 @@ def test_add_column_foreign_key(db_path):
assert "table 'bobcats' does not exist" in str(result.exception)
def test_suggest_alter_if_column_missing(db_path):
db = Database(db_path)
db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "authors", "-"],
input='{"id": 2, "name": "Barry", "age": 43}',
)
assert result.exit_code != 0
assert result.output.strip() == (
"Error: table authors has no column named age\n\n"
"Try using --alter to add additional columns"
)
def test_index_foreign_keys(db_path):
test_add_column_foreign_key(db_path)
db = Database(db_path)
@ -367,7 +410,7 @@ def test_index_foreign_keys(db_path):
def test_enable_fts(db_path):
db = Database(db_path)
assert None == db["Gosh"].detect_fts()
assert db["Gosh"].detect_fts() is None
result = CliRunner().invoke(
cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
)
@ -377,7 +420,7 @@ def test_enable_fts(db_path):
# Table names with restricted chars are handled correctly.
# colons and dots are restricted characters for table names.
db["http://example.com"].create({"c1": str, "c2": str, "c3": str})
assert None == db["http://example.com"].detect_fts()
assert db["http://example.com"].detect_fts() is None
result = CliRunner().invoke(
cli.cli,
[
@ -476,6 +519,13 @@ def test_vacuum(db_path):
assert 0 == result.exit_code
def test_dump(db_path):
result = CliRunner().invoke(cli.cli, ["dump", db_path])
assert result.exit_code == 0
assert result.output.startswith("BEGIN TRANSACTION;")
assert result.output.strip().endswith("COMMIT;")
@pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"]))
def test_optimize(db_path, tables):
db = Database(db_path)
@ -563,8 +613,8 @@ def test_insert_simple(tmpdir):
open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4}))
result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path])
assert 0 == result.exit_code
assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts(
"select * from dogs"
assert [{"age": 4, "name": "Cleo"}] == list(
Database(db_path).query("select * from dogs")
)
db = Database(db_path)
assert ["dogs"] == db.table_names()
@ -579,8 +629,8 @@ def test_insert_from_stdin(tmpdir):
input=json.dumps({"name": "Cleo", "age": 4}),
)
assert 0 == result.exit_code
assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts(
"select * from dogs"
assert [{"age": 4, "name": "Cleo"}] == list(
Database(db_path).query("select * from dogs")
)
@ -598,6 +648,34 @@ def test_insert_invalid_json_error(tmpdir):
)
def test_insert_json_flatten(tmpdir):
db_path = str(tmpdir / "flat.db")
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "items", "-", "--flatten"],
input=json.dumps({"nested": {"data": 4}}),
)
assert result.exit_code == 0
assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}]
def test_insert_json_flatten_nl(tmpdir):
db_path = str(tmpdir / "flat.db")
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "items", "-", "--flatten", "--nl"],
input="\n".join(
json.dumps(item)
for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}]
),
)
assert result.exit_code == 0
assert list(Database(db_path).query("select * from items")) == [
{"nested_data": 4, "nested_other": None},
{"nested_data": None, "nested_other": 3},
]
def test_insert_with_primary_key(db_path, tmpdir):
json_path = str(tmpdir / "dog.json")
open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4}))
@ -605,9 +683,9 @@ def test_insert_with_primary_key(db_path, tmpdir):
cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
)
assert 0 == result.exit_code
assert [{"id": 1, "age": 4, "name": "Cleo"}] == Database(
db_path
).execute_returning_dicts("select * from dogs")
assert [{"id": 1, "age": 4, "name": "Cleo"}] == list(
Database(db_path).query("select * from dogs")
)
db = Database(db_path)
assert ["id"] == db["dogs"].pks
@ -621,7 +699,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir):
)
assert 0 == result.exit_code
db = Database(db_path)
assert dogs == db.execute_returning_dicts("select * from dogs order by id")
assert dogs == list(db.query("select * from dogs order by id"))
assert ["id"] == db["dogs"].pks
@ -637,7 +715,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir):
)
assert 0 == result.exit_code
db = Database(db_path)
assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id")
assert dogs == list(db.query("select * from dogs order by breed, id"))
assert {"breed", "id"} == set(db["dogs"].pks)
assert (
"CREATE TABLE [dogs] (\n"
@ -682,7 +760,7 @@ def test_insert_binary_base64(db_path):
)
assert 0 == result.exit_code, result.output
db = Database(db_path)
actual = db.execute_returning_dicts("select content from files")
actual = list(db.query("select content from files"))
assert actual == [{"content": b"hello"}]
@ -697,7 +775,7 @@ def test_insert_newline_delimited(db_path):
assert [
{"foo": "bar", "n": 1},
{"foo": "baz", "n": 2},
] == db.execute_returning_dicts("select foo, n from from_json_nl")
] == list(db.query("select foo, n from from_json_nl"))
def test_insert_ignore(db_path, tmpdir):
@ -716,9 +794,7 @@ def test_insert_ignore(db_path, tmpdir):
)
assert 0 == result.exit_code, result.output
# ... but it should actually have no effect
assert [{"id": 1, "name": "Cleo"}] == db.execute_returning_dicts(
"select * from dogs"
)
assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs"))
@pytest.mark.parametrize(
@ -781,8 +857,9 @@ def test_insert_replace(db_path, tmpdir):
)
assert 0 == result.exit_code, result.output
assert 21 == db["dogs"].count
assert insert_replace_dogs == db.execute_returning_dicts(
"select * from dogs where id in (1, 2, 21) order by id"
assert (
list(db.query("select * from dogs where id in (1, 2, 21) order by id"))
== insert_replace_dogs
)
@ -797,7 +874,7 @@ def test_insert_truncate(db_path):
assert [
{"foo": "bar", "n": 1},
{"foo": "baz", "n": 2},
] == db.execute_returning_dicts("select foo, n from from_json_nl")
] == list(db.query("select foo, n from from_json_nl"))
# Truncate and insert new rows
result = CliRunner().invoke(
cli.cli,
@ -816,7 +893,7 @@ def test_insert_truncate(db_path):
assert [
{"foo": "bam", "n": 3},
{"foo": "bat", "n": 4},
] == db.execute_returning_dicts("select foo, n from from_json_nl")
] == list(db.query("select foo, n from from_json_nl"))
def test_insert_alter(db_path, tmpdir):
@ -847,7 +924,7 @@ def test_insert_alter(db_path, tmpdir):
{"foo": "bar", "n": 1, "baz": None},
{"foo": "baz", "n": 2, "baz": None},
{"foo": "bar", "baz": 5, "n": None},
] == db.execute_returning_dicts("select foo, n, baz from from_json_nl")
] == list(db.query("select foo, n, baz from from_json_nl"))
@pytest.mark.parametrize(
@ -917,7 +994,23 @@ def test_query_json(db_path, sql, args, expected):
assert expected == result.output.strip()
LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f<c\xc9\xf5L\x0f\xd7E\xad/\x9b\x9eI^2\x93\x1a\x9b\xf6F^\n\xd7\xd4\x8f\xca\xfb\x90.\xdd/\xfd\x94\xd4\x11\x87I8\x1a\xaf\xd1S?\x06\x88\xa7\xecBo\xbb$\xbb\t\xe9\xf4\xe8\xe4\x98U\x1bM\x19S\xbe\xa4e\x991x\xfcx\xf6\xe2#\x9e\x93h'&%YK(i)\x7f\t\xc5@N7\xbf+\x1b\xb5\xdd\x10\r\x9e\xb1\xf0y\xa1\xf7W\x92a\xe2;\xc6\xc8\xa0\xa7\xc4\x92\xe2\\\xf2\xa1\x99m\xdf\x88)\xc6\xec\x9a\xa5\xed\x14wR\xf1h\xf22x\xcfM\xfdv\xd3\xa4LY\x96\xcc\xbd[{\xd9m\xf0\x0eH#\x8e\xf5\x9b\xab\xd7\xcb\xe9t\x05\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}"
LOREM_IPSUM_COMPRESSED = (
b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e"
b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J"
b"\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b"
b"$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3"
b"\x85>\x8c\xa4i\x8d\xdaTu\x7f<c\xc9\xf5L\x0f\xd7E\xad/\x9b\x9eI^2\x93\x1a\x9b"
b"\xf6F^\n\xd7\xd4\x8f\xca\xfb\x90.\xdd/\xfd\x94\xd4\x11\x87I8\x1a\xaf\xd1S?\x06"
b"\x88\xa7\xecBo\xbb$\xbb\t\xe9\xf4\xe8\xe4\x98U\x1bM\x19S\xbe\xa4e\x991x\xfc"
b"x\xf6\xe2#\x9e\x93h'&%YK(i)\x7f\t\xc5@N7\xbf+\x1b\xb5\xdd\x10\r\x9e\xb1\xf0"
b"y\xa1\xf7W\x92a\xe2;\xc6\xc8\xa0\xa7\xc4\x92\xe2\\\xf2\xa1\x99m\xdf\x88)\xc6"
b"\xec\x9a\xa5\xed\x14wR\xf1h\xf22x\xcfM\xfdv\xd3\xa4LY\x96\xcc\xbd[{\xd9m\xf0"
b"\x0eH#\x8e\xf5\x9b\xab\xd7\xcb\xe9t\x05\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03"
b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}"
)
def test_query_json_binary(db_path):
@ -939,7 +1032,18 @@ def test_query_json_binary(db_path):
"sz": 16984,
"data": {
"$base64": True,
"encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9",
"encoded": (
(
"eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH"
"8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+"
"DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I"
"/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI"
"jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f"
"iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8"
"IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A"
"Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9"
)
),
},
}
]
@ -1023,7 +1127,7 @@ def test_query_load_extension(use_spatialite_shortcut):
# Without --load-extension:
result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"])
assert result.exit_code == 1
assert "no such function: spatialite_version" in repr(result)
assert "no such function: spatialite_version" in result.output
# With --load-extension:
if use_spatialite_shortcut:
load_extension = "spatialite"
@ -1108,17 +1212,32 @@ def test_upsert(db_path, tmpdir):
{"id": 1, "age": 5},
{"id": 2, "age": 5},
]
open(json_path, "w").write(json.dumps(insert_dogs))
open(json_path, "w").write(json.dumps(upsert_dogs))
result = CliRunner().invoke(
cli.cli,
["upsert", db_path, "dogs", json_path, "--pk", "id"],
catch_exceptions=False,
)
assert 0 == result.exit_code, result.output
assert [
{"id": 1, "name": "Cleo", "age": 4},
{"id": 2, "name": "Nixie", "age": 4},
] == db.execute_returning_dicts("select * from dogs order by id")
assert list(db.query("select * from dogs order by id")) == [
{"id": 1, "name": "Cleo", "age": 5},
{"id": 2, "name": "Nixie", "age": 5},
]
def test_upsert_flatten(tmpdir):
db_path = str(tmpdir / "flat.db")
db = Database(db_path)
db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id")
result = CliRunner().invoke(
cli.cli,
["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"],
input=json.dumps({"id": 1, "nested": {"two": 2}}),
)
assert result.exit_code == 0
assert list(db.query("select * from upsert_me")) == [
{"id": 1, "name": "Example", "nested_two": 2}
]
def test_upsert_alter(db_path, tmpdir):
@ -1137,7 +1256,11 @@ def test_upsert_alter(db_path, tmpdir):
cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"]
)
assert 1 == result.exit_code
assert "no such column: age" == str(result.exception)
assert (
"Error: no such column: age\n\n"
"sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n"
"parameters = [5, 1]"
) == result.output.strip()
# Should succeed with --alter
result = CliRunner().invoke(
cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"]
@ -1145,7 +1268,7 @@ def test_upsert_alter(db_path, tmpdir):
assert 0 == result.exit_code
assert [
{"id": 1, "name": "Cleo", "age": 5},
] == db.execute_returning_dicts("select * from dogs order by id")
] == list(db.query("select * from dogs order by id"))
@pytest.mark.parametrize(
@ -1499,7 +1622,7 @@ def test_query_update(db_path, args, expected):
cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args
)
assert expected == result.output.strip()
assert db.execute_returning_dicts("select * from dogs") == [
assert list(db.query("select * from dogs")) == [
{"id": 1, "age": 5, "name": "Cleo"},
]
@ -1547,47 +1670,112 @@ def test_add_foreign_keys(db_path):
[
(
[],
"CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT\n"
")"
),
),
(
["--type", "age", "text"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] TEXT NOT NULL DEFAULT '1',\n [name] TEXT\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] TEXT NOT NULL DEFAULT '1',\n"
" [name] TEXT\n"
")"
),
),
(
["--drop", "age"],
'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)',
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [name] TEXT\n"
")"
),
),
(
["--rename", "age", "age2", "--rename", "id", "pk"],
"CREATE TABLE \"dogs\" (\n [pk] INTEGER PRIMARY KEY,\n [age2] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)",
(
'CREATE TABLE "dogs" (\n'
" [pk] INTEGER PRIMARY KEY,\n"
" [age2] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT\n"
")"
),
),
(
["--not-null", "name"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT NOT NULL\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT NOT NULL\n"
")"
),
),
(
["--not-null-false", "age"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER DEFAULT '1',\n [name] TEXT\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] INTEGER DEFAULT '1',\n"
" [name] TEXT\n"
")"
),
),
(
["--pk", "name"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT PRIMARY KEY\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT PRIMARY KEY\n"
")"
),
),
(
["--pk-none"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT\n"
")"
),
),
(
["--default", "name", "Turnip"],
"CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT DEFAULT 'Turnip'\n)",
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [name] TEXT DEFAULT 'Turnip'\n"
")"
),
),
(
["--default-none", "age"],
'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)',
(
'CREATE TABLE "dogs" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [age] INTEGER NOT NULL,\n"
" [name] TEXT\n"
")"
),
),
(
["-o", "name", "--column-order", "age", "-o", "id"],
"CREATE TABLE \"dogs\" (\n [name] TEXT,\n [age] INTEGER NOT NULL DEFAULT '1',\n [id] INTEGER PRIMARY KEY\n)",
(
'CREATE TABLE "dogs" (\n'
" [name] TEXT,\n"
" [age] INTEGER NOT NULL DEFAULT '1',\n"
" [id] INTEGER PRIMARY KEY\n"
")"
),
),
],
)
@ -1636,9 +1824,13 @@ def test_transform_drop_foreign_key(db_path):
print(result.output)
assert result.exit_code == 0
schema = db["places"].schema
assert (
schema
== 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)'
assert schema == (
'CREATE TABLE "places" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [name] TEXT,\n"
" [country] INTEGER,\n"
" [city] INTEGER REFERENCES [city]([id])\n"
")"
)
@ -1652,22 +1844,48 @@ _common_other_schema = (
[
(
[],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
(
'CREATE TABLE "trees" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [address] TEXT,\n"
" [species_id] INTEGER,\n"
" FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
")"
),
_common_other_schema,
),
(
["--table", "custom_table"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)',
(
'CREATE TABLE "trees" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [address] TEXT,\n"
" [custom_table_id] INTEGER,\n"
" FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n"
")"
),
"CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)",
),
(
["--fk-column", "custom_fk"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)',
(
'CREATE TABLE "trees" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [address] TEXT,\n"
" [custom_fk] INTEGER,\n"
" FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n"
")"
),
_common_other_schema,
),
(
["--rename", "name", "name2"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
'CREATE TABLE "trees" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [address] TEXT,\n"
" [species_id] INTEGER,\n"
" FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
")",
"CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)",
),
],
@ -1776,7 +1994,87 @@ def test_search(tmpdir, fts, extra_arg, expected):
assert result.output.replace("\r", "") == expected
_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n'
def test_indexes(tmpdir):
db_path = str(tmpdir / "test.db")
db = Database(db_path)
db.conn.executescript(
"""
create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_idx on Gosh(c2, c3 desc);
"""
)
result = CliRunner().invoke(
cli.cli,
["indexes", str(db_path)],
catch_exceptions=False,
)
assert result.exit_code == 0
assert json.loads(result.output) == [
{
"table": "Gosh",
"index_name": "Gosh_idx",
"seqno": 0,
"cid": 1,
"name": "c2",
"desc": 0,
"coll": "BINARY",
"key": 1,
},
{
"table": "Gosh",
"index_name": "Gosh_idx",
"seqno": 1,
"cid": 2,
"name": "c3",
"desc": 1,
"coll": "BINARY",
"key": 1,
},
]
result2 = CliRunner().invoke(
cli.cli,
["indexes", str(db_path), "--aux"],
catch_exceptions=False,
)
assert result2.exit_code == 0
assert json.loads(result2.output) == [
{
"table": "Gosh",
"index_name": "Gosh_idx",
"seqno": 0,
"cid": 1,
"name": "c2",
"desc": 0,
"coll": "BINARY",
"key": 1,
},
{
"table": "Gosh",
"index_name": "Gosh_idx",
"seqno": 1,
"cid": 2,
"name": "c3",
"desc": 1,
"coll": "BINARY",
"key": 1,
},
{
"table": "Gosh",
"index_name": "Gosh_idx",
"seqno": 2,
"cid": -1,
"name": None,
"desc": 0,
"coll": "BINARY",
"key": 0,
},
]
_TRIGGERS_EXPECTED = (
'[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah '
'AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n'
)
@pytest.mark.parametrize(
@ -1813,6 +2111,60 @@ def test_triggers(tmpdir, extra_args, expected):
assert result.output == expected
@pytest.mark.parametrize(
"options,expected",
(
(
[],
(
"CREATE TABLE [dogs] (\n"
" [id] INTEGER,\n"
" [name] TEXT\n"
");\n"
"CREATE TABLE [chickens] (\n"
" [id] INTEGER,\n"
" [name] TEXT,\n"
" [breed] TEXT\n"
");\n"
"CREATE INDEX [idx_chickens_breed]\n"
" ON [chickens] ([breed]);\n"
),
),
(
["dogs"],
("CREATE TABLE [dogs] (\n" " [id] INTEGER,\n" " [name] TEXT\n" ")\n"),
),
(
["chickens", "dogs"],
(
"CREATE TABLE [chickens] (\n"
" [id] INTEGER,\n"
" [name] TEXT,\n"
" [breed] TEXT\n"
")\n"
"CREATE TABLE [dogs] (\n"
" [id] INTEGER,\n"
" [name] TEXT\n"
")\n"
),
),
),
)
def test_schema(tmpdir, options, expected):
db_path = str(tmpdir / "test.db")
db = Database(db_path)
db["dogs"].create({"id": int, "name": str})
db["chickens"].create({"id": int, "name": str, "breed": str})
db["chickens"].create_index(["breed"])
result = CliRunner().invoke(
cli.cli,
["schema", db_path] + options,
catch_exceptions=False,
)
assert result.exit_code == 0
assert result.output == expected
def test_long_csv_column_value(tmpdir):
db_path = str(tmpdir / "test.db")
csv_path = str(tmpdir / "test.csv")
@ -1889,3 +2241,85 @@ def test_attach(tmpdir):
{"id": 1, "text": "foo"},
{"id": 1, "text": "bar"},
]
def test_csv_insert_bom(tmpdir):
db_path = str(tmpdir / "test.db")
bom_csv_path = str(tmpdir / "bom.csv")
with open(bom_csv_path, "wb") as fp:
fp.write(b"\xef\xbb\xbfname,age\nCleo,5")
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"],
catch_exceptions=False,
)
assert result.exit_code == 0
result2 = CliRunner().invoke(
cli.cli,
["insert", db_path, "fixed", bom_csv_path, "--csv"],
catch_exceptions=False,
)
assert result2.exit_code == 0
db = Database(db_path)
tables = db.execute("select name, sql from sqlite_master").fetchall()
assert tables == [
("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"),
("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"),
]
@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types"))
def test_insert_detect_types(tmpdir, option_or_env_var):
db_path = str(tmpdir / "test.db")
data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5"
extra = []
if option_or_env_var:
extra = [option_or_env_var]
def _test():
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "creatures", "-", "--csv"] + extra,
catch_exceptions=False,
input=data,
)
assert result.exit_code == 0
db = Database(db_path)
assert list(db["creatures"].rows) == [
{"name": "Cleo", "age": 6, "weight": 45.5},
{"name": "Dori", "age": 1, "weight": 3.5},
]
if option_or_env_var is None:
# Use environemnt variable instead of option
with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}):
_test()
else:
_test()
@pytest.mark.parametrize(
"input,expected",
(
({"foo": {"bar": 1}}, {"foo_bar": 1}),
({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}),
({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}),
),
)
def test_flatten_helper(input, expected):
assert dict(cli._flatten(input)) == expected
def test_integer_overflow_error(tmpdir):
db_path = str(tmpdir / "test.db")
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "items", "-"],
input=json.dumps({"bignumber": 34223049823094832094802398430298048240}),
)
assert result.exit_code == 1
assert result.output == (
"Error: Python int too large to convert to SQLite INTEGER\n\n"
"sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n"
"parameters = [34223049823094832094802398430298048240]\n"
)

517
tests/test_cli_convert.py Normal file
View file

@ -0,0 +1,517 @@
from click.testing import CliRunner
from sqlite_utils import cli
import sqlite_utils
import json
import textwrap
import pathlib
import pytest
@pytest.fixture
def test_db_and_path(fresh_db_and_path):
db, db_path = fresh_db_and_path
db["example"].insert_all(
[
{"id": 1, "dt": "5th October 2019 12:04"},
{"id": 2, "dt": "6th October 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
],
pk="id",
)
return db, db_path
@pytest.fixture
def fresh_db_and_path(tmpdir):
db_path = str(pathlib.Path(tmpdir) / "data.db")
db = sqlite_utils.Database(db_path)
return db, db_path
@pytest.mark.parametrize(
"code",
[
"return value.replace('October', 'Spooktober')",
# Return is optional:
"value.replace('October', 'Spooktober')",
],
)
def test_convert_single_line(test_db_and_path, code):
db, db_path = test_db_and_path
result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code])
assert 0 == result.exit_code, result.output
assert [
{"id": 1, "dt": "5th Spooktober 2019 12:04"},
{"id": 2, "dt": "6th Spooktober 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
] == list(db["example"].rows)
def test_convert_multiple_lines(test_db_and_path):
db, db_path = test_db_and_path
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"dt",
"v = value.replace('October', 'Spooktober')\nreturn v.upper()",
],
)
assert 0 == result.exit_code, result.output
assert [
{"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"},
{"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
] == list(db["example"].rows)
def test_convert_import(test_db_and_path):
db, db_path = test_db_and_path
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"dt",
"return re.sub('O..', 'OXX', value)",
"--import",
"re",
],
)
assert 0 == result.exit_code, result.output
assert [
{"id": 1, "dt": "5th OXXober 2019 12:04"},
{"id": 2, "dt": "6th OXXober 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
] == list(db["example"].rows)
def test_convert_dryrun(test_db_and_path):
db, db_path = test_db_and_path
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"dt",
"return re.sub('O..', 'OXX', value)",
"--import",
"re",
"--dry-run",
],
)
assert result.exit_code == 0
assert result.output.strip() == (
"5th October 2019 12:04\n"
" --- becomes:\n"
"5th OXXober 2019 12:04\n"
"\n"
"6th October 2019 00:05:06\n"
" --- becomes:\n"
"6th OXXober 2019 00:05:06\n"
"\n"
"\n"
" --- becomes:\n"
"\n"
"\n"
"None\n"
" --- becomes:\n"
"None\n\n"
"Would affect 4 rows"
)
# But it should not have actually modified the table data
assert list(db["example"].rows) == [
{"id": 1, "dt": "5th October 2019 12:04"},
{"id": 2, "dt": "6th October 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
]
# Test with a where clause too
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"dt",
"return re.sub('O..', 'OXX', value)",
"--import",
"re",
"--dry-run",
"--where",
"id = :id",
"-p",
"id",
"4",
],
)
assert result.exit_code == 0
assert result.output.strip().split("\n")[-1] == "Would affect 1 row"
@pytest.mark.parametrize("drop", (True, False))
def test_convert_output_column(test_db_and_path, drop):
db, db_path = test_db_and_path
args = [
"convert",
db_path,
"example",
"dt",
"value.replace('October', 'Spooktober')",
"--output",
"newcol",
]
if drop:
args += ["--drop"]
result = CliRunner().invoke(cli.cli, args)
assert 0 == result.exit_code, result.output
expected = [
{
"id": 1,
"dt": "5th October 2019 12:04",
"newcol": "5th Spooktober 2019 12:04",
},
{
"id": 2,
"dt": "6th October 2019 00:05:06",
"newcol": "6th Spooktober 2019 00:05:06",
},
{"id": 3, "dt": "", "newcol": ""},
{"id": 4, "dt": None, "newcol": None},
]
if drop:
for row in expected:
del row["dt"]
assert list(db["example"].rows) == expected
@pytest.mark.parametrize(
"output_type,expected",
(
("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]),
("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]),
(None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]),
),
)
def test_convert_output_column_output_type(test_db_and_path, output_type, expected):
db, db_path = test_db_and_path
args = [
"convert",
db_path,
"example",
"id",
"value",
"--output",
"new_id",
]
if output_type:
args += ["--output-type", output_type]
result = CliRunner().invoke(
cli.cli,
args,
)
assert 0 == result.exit_code, result.output
assert expected == list(db.execute("select id, new_id from example"))
@pytest.mark.parametrize(
"options,expected_error",
[
(
[
"dt",
"id",
"value.replace('October', 'Spooktober')",
"--output",
"newcol",
],
"Cannot use --output with more than one column",
),
(
[
"dt",
"value.replace('October', 'Spooktober')",
"--output",
"newcol",
"--output-type",
"invalid",
],
"Error: Invalid value for '--output-type'",
),
(
[
"value.replace('October', 'Spooktober')",
],
"Missing argument 'COLUMNS...'",
),
],
)
def test_convert_output_error(test_db_and_path, options, expected_error):
db_path = test_db_and_path[1]
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
]
+ options,
)
assert result.exit_code != 0
assert expected_error in result.output
@pytest.mark.parametrize("drop", (True, False))
def test_convert_multi(fresh_db_and_path, drop):
db, db_path = fresh_db_and_path
db["creatures"].insert_all(
[
{"id": 1, "name": "Simon"},
{"id": 2, "name": "Cleo"},
],
pk="id",
)
args = [
"convert",
db_path,
"creatures",
"name",
"--multi",
'{"upper": value.upper(), "lower": value.lower()}',
]
if drop:
args += ["--drop"]
result = CliRunner().invoke(cli.cli, args)
assert result.exit_code == 0, result.output
expected = [
{"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"},
{"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"},
]
if drop:
for row in expected:
del row["name"]
assert list(db["creatures"].rows) == expected
def test_convert_multi_complex_column_types(fresh_db_and_path):
db, db_path = fresh_db_and_path
db["rows"].insert_all(
[
{"id": 1},
{"id": 2},
{"id": 3},
{"id": 4},
],
pk="id",
)
code = textwrap.dedent(
"""
if value == 1:
return {"is_str": "", "is_float": 1.2, "is_int": None}
elif value == 2:
return {"is_float": 1, "is_int": 12}
elif value == 3:
return {"is_bytes": b"blah"}
"""
)
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"rows",
"id",
"--multi",
code,
],
)
assert result.exit_code == 0, result.output
assert list(db["rows"].rows) == [
{"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None},
{"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None},
{
"id": 3,
"is_str": None,
"is_float": None,
"is_int": None,
"is_bytes": b"blah",
},
{"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None},
]
assert db["rows"].schema == (
"CREATE TABLE [rows] (\n"
" [id] INTEGER PRIMARY KEY\n"
", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)"
)
@pytest.mark.parametrize("delimiter", [None, ";", "-"])
def test_recipe_jsonsplit(tmpdir, delimiter):
db_path = str(pathlib.Path(tmpdir) / "data.db")
db = sqlite_utils.Database(db_path)
db["example"].insert_all(
[
{"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
{"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
],
pk="id",
)
code = "r.jsonsplit(value)"
if delimiter:
code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter)
args = ["convert", db_path, "example", "tags", code]
result = CliRunner().invoke(cli.cli, args)
assert 0 == result.exit_code, result.output
assert list(db["example"].rows) == [
{"id": 1, "tags": '["foo", "bar"]'},
{"id": 2, "tags": '["bar", "baz"]'},
]
@pytest.mark.parametrize(
"type,expected_array",
(
(None, ["1", "2", "3"]),
("float", [1.0, 2.0, 3.0]),
("int", [1, 2, 3]),
),
)
def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array):
db, db_path = fresh_db_and_path
db["example"].insert_all(
[
{"id": 1, "records": "1,2,3"},
],
pk="id",
)
code = "r.jsonsplit(value)"
if type:
code = "recipes.jsonsplit(value, type={})".format(type)
args = ["convert", db_path, "example", "records", code]
result = CliRunner().invoke(cli.cli, args)
assert 0 == result.exit_code, result.output
assert json.loads(db["example"].get(1)["records"]) == expected_array
@pytest.mark.parametrize("drop", (True, False))
def test_recipe_jsonsplit_output(fresh_db_and_path, drop):
db, db_path = fresh_db_and_path
db["example"].insert_all(
[
{"id": 1, "records": "1,2,3"},
],
pk="id",
)
code = "r.jsonsplit(value)"
args = ["convert", db_path, "example", "records", code, "--output", "tags"]
if drop:
args += ["--drop"]
result = CliRunner().invoke(cli.cli, args)
assert 0 == result.exit_code, result.output
expected = {
"id": 1,
"records": "1,2,3",
"tags": '["1", "2", "3"]',
}
if drop:
del expected["records"]
assert db["example"].get(1) == expected
def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path):
args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"]
result = CliRunner().invoke(cli.cli, args)
assert result.exit_code == 1, result.output
assert "Error: --drop can only be used with --output or --multi" in result.output
def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path):
args = [
"convert",
fresh_db_and_path[1],
"example",
"records",
"othercol",
"value",
"--multi",
]
result = CliRunner().invoke(cli.cli, args)
assert result.exit_code == 1, result.output
assert "Error: Cannot use --multi with more than one column" in result.output
def test_multi_with_bad_function(test_db_and_path):
args = [
"convert",
test_db_and_path[1],
"example",
"dt",
"value.upper()",
"--multi",
]
result = CliRunner().invoke(cli.cli, args)
assert result.exit_code == 1, result.output
assert "When using --multi code must return a Python dictionary" in result.output
def test_convert_where(test_db_and_path):
db, db_path = test_db_and_path
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"dt",
"str(value).upper()",
"--where",
"id = :id",
"-p",
"id",
2,
],
)
assert result.exit_code == 0, result.output
assert list(db["example"].rows) == [
{"id": 1, "dt": "5th October 2019 12:04"},
{"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
]
def test_convert_where_multi(fresh_db_and_path):
db, db_path = fresh_db_and_path
db["names"].insert_all(
[{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id"
)
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"names",
"name",
'{"upper": value.upper()}',
"--where",
"id = :id",
"-p",
"id",
2,
"--multi",
],
)
assert 0 == result.exit_code, result.output
assert list(db["names"].rows) == [
{"id": 1, "name": "Cleo", "upper": None},
{"id": 2, "name": "Bants", "upper": "BANTS"},
]

222
tests/test_cli_memory.py Normal file
View file

@ -0,0 +1,222 @@
import json
import pytest
from click.testing import CliRunner
from sqlite_utils import Database, cli
def test_memory_basic():
result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"])
assert result.exit_code == 0
assert result.output.strip() == '[{"1 + 1": 2}]'
@pytest.mark.parametrize("sql_from", ("test", "t", "t1"))
@pytest.mark.parametrize("use_stdin", (True, False))
def test_memory_csv(tmpdir, sql_from, use_stdin):
content = "id,name\n1,Cleo\n2,Bants"
input = None
if use_stdin:
input = content
csv_path = "-"
if sql_from == "test":
sql_from = "stdin"
else:
csv_path = str(tmpdir / "test.csv")
open(csv_path, "w").write(content)
result = CliRunner().invoke(
cli.cli,
["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
input=input,
)
assert result.exit_code == 0
assert (
result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}'
)
@pytest.mark.parametrize("use_stdin", (True, False))
def test_memory_tsv(tmpdir, use_stdin):
data = "id\tname\n1\tCleo\n2\tBants"
if use_stdin:
input = data
path = "stdin:tsv"
sql_from = "stdin"
else:
input = None
path = str(tmpdir / "chickens.tsv")
open(path, "w").write(data)
path = path + ":tsv"
sql_from = "chickens"
result = CliRunner().invoke(
cli.cli,
["memory", path, "select * from {}".format(sql_from)],
input=input,
)
assert result.exit_code == 0, result.output
assert json.loads(result.output.strip()) == [
{"id": 1, "name": "Cleo"},
{"id": 2, "name": "Bants"},
]
@pytest.mark.parametrize("use_stdin", (True, False))
def test_memory_json(tmpdir, use_stdin):
data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]'
if use_stdin:
input = data
path = "stdin:json"
sql_from = "stdin"
else:
input = None
path = str(tmpdir / "chickens.json")
open(path, "w").write(data)
path = path + ":json"
sql_from = "chickens"
result = CliRunner().invoke(
cli.cli,
["memory", path, "select * from {}".format(sql_from)],
input=input,
)
assert result.exit_code == 0, result.output
assert json.loads(result.output.strip()) == [
{"name": "Bants", "age": None, "nested": None},
{"name": "Dori", "age": 1, "nested": '{"nest": 1}'},
]
@pytest.mark.parametrize("use_stdin", (True, False))
def test_memory_json_nl(tmpdir, use_stdin):
data = '{"name": "Bants"}\n\n{"name": "Dori"}'
if use_stdin:
input = data
path = "stdin:nl"
sql_from = "stdin"
else:
input = None
path = str(tmpdir / "chickens.json")
open(path, "w").write(data)
path = path + ":nl"
sql_from = "chickens"
result = CliRunner().invoke(
cli.cli,
["memory", path, "select * from {}".format(sql_from)],
input=input,
)
assert result.exit_code == 0, result.output
assert json.loads(result.output.strip()) == [
{"name": "Bants"},
{"name": "Dori"},
]
@pytest.mark.parametrize("use_stdin", (True, False))
def test_memory_csv_encoding(tmpdir, use_stdin):
latin1_csv = (
b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n"
)
input = None
if use_stdin:
input = latin1_csv
csv_path = "-"
sql_from = "stdin"
else:
csv_path = str(tmpdir / "test.csv")
with open(csv_path, "wb") as fp:
fp.write(latin1_csv)
sql_from = "test"
# Without --encoding should error:
assert (
CliRunner()
.invoke(
cli.cli,
["memory", csv_path, "select * from {}".format(sql_from), "--nl"],
input=input,
)
.exit_code
== 1
)
# With --encoding should work:
result = CliRunner().invoke(
cli.cli,
["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"],
input=latin1_csv,
)
assert result.exit_code == 0, result.output
assert json.loads(result.output.strip()) == {
"date": "2020-03-04",
"name": "São Paulo",
"latitude": -23.561,
"longitude": -46.645,
}
@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
def test_memory_dump(extra_args):
result = CliRunner().invoke(
cli.cli,
["memory", "-"] + extra_args + ["--dump"],
input="id,name\n1,Cleo\n2,Bants",
)
assert result.exit_code == 0
assert result.output.strip() == (
"BEGIN TRANSACTION;\n"
'CREATE TABLE "stdin" (\n'
" [id] INTEGER,\n"
" [name] TEXT\n"
");\n"
"INSERT INTO \"stdin\" VALUES(1,'Cleo');\n"
"INSERT INTO \"stdin\" VALUES(2,'Bants');\n"
"CREATE VIEW t1 AS select * from [stdin];\n"
"CREATE VIEW t AS select * from [stdin];\n"
"COMMIT;"
)
@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
def test_memory_schema(extra_args):
result = CliRunner().invoke(
cli.cli,
["memory", "-"] + extra_args + ["--schema"],
input="id,name\n1,Cleo\n2,Bants",
)
assert result.exit_code == 0
assert result.output.strip() == (
'CREATE TABLE "stdin" (\n'
" [id] INTEGER,\n"
" [name] TEXT\n"
");\n"
"CREATE VIEW t1 AS select * from [stdin];\n"
"CREATE VIEW t AS select * from [stdin];"
)
@pytest.mark.parametrize("extra_args", ([], ["select 1"]))
def test_memory_save(tmpdir, extra_args):
save_to = str(tmpdir / "save.db")
result = CliRunner().invoke(
cli.cli,
["memory", "-"] + extra_args + ["--save", save_to],
input="id,name\n1,Cleo\n2,Bants",
)
assert result.exit_code == 0
db = Database(save_to)
assert list(db["stdin"].rows) == [
{"id": 1, "name": "Cleo"},
{"id": 2, "name": "Bants"},
]
@pytest.mark.parametrize("option", ("-n", "--no-detect-types"))
def test_memory_no_detect_types(option):
result = CliRunner().invoke(
cli.cli,
["memory", "-", "select * from stdin"] + [option],
input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5",
)
assert result.exit_code == 0, result.output
assert json.loads(result.output.strip()) == [
{"id": "1", "name": "Cleo", "weight": "45.5"},
{"id": "2", "name": "Bants", "weight": "3.5"},
]

1
tests/test_constructor.py
View file

@ -1,5 +1,4 @@
from sqlite_utils import Database
import pytest
def test_recursive_triggers():

3
tests/test_conversions.py
View file

@ -1,6 +1,3 @@
import pytest
def test_insert_conversion(fresh_db):
table = fresh_db["table"]
table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"})

117
tests/test_convert.py Normal file
View file

@ -0,0 +1,117 @@
from sqlite_utils.db import BadMultiValues
import pytest
@pytest.mark.parametrize(
"columns,fn,expected",
(
(
"title",
lambda value: value.upper(),
{"title": "MIXED CASE", "abstract": "Abstract"},
),
(
["title", "abstract"],
lambda value: value.upper(),
{"title": "MIXED CASE", "abstract": "ABSTRACT"},
),
),
)
def test_convert(fresh_db, columns, fn, expected):
table = fresh_db["table"]
table.insert({"title": "Mixed Case", "abstract": "Abstract"})
table.convert(columns, fn)
assert list(table.rows) == [expected]
@pytest.mark.parametrize(
"where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1]))
)
def test_convert_where(fresh_db, where, where_args):
table = fresh_db["table"]
table.insert_all(
[
{"id": 1, "title": "One"},
{"id": 2, "title": "Two"},
],
pk="id",
)
table.convert(
"title", lambda value: value.upper(), where=where, where_args=where_args
)
assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}]
@pytest.mark.parametrize(
"drop,expected",
(
(False, {"title": "Mixed Case", "other": "MIXED CASE"}),
(True, {"other": "MIXED CASE"}),
),
)
def test_convert_output(fresh_db, drop, expected):
table = fresh_db["table"]
table.insert({"title": "Mixed Case"})
table.convert("title", lambda v: v.upper(), output="other", drop=drop)
assert list(table.rows) == [expected]
def test_convert_output_multiple_column_error(fresh_db):
table = fresh_db["table"]
with pytest.raises(AssertionError) as excinfo:
table.convert(["title", "other"], lambda v: v, output="out")
assert "output= can only be used with a single column" in str(excinfo.value)
@pytest.mark.parametrize(
"type,expected",
(
(int, {"other": 123}),
(float, {"other": 123.0}),
),
)
def test_convert_output_type(fresh_db, type, expected):
table = fresh_db["table"]
table.insert({"number": "123"})
table.convert("number", lambda v: v, output="other", output_type=type, drop=True)
assert list(table.rows) == [expected]
def test_convert_multi(fresh_db):
table = fresh_db["table"]
table.insert({"title": "Mixed Case"})
table.convert(
"title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True
)
assert list(table.rows) == [
{"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"}
]
def test_convert_multi_where(fresh_db):
table = fresh_db["table"]
table.insert_all(
[
{"id": 1, "title": "One"},
{"id": 2, "title": "Two"},
],
pk="id",
)
table.convert(
"title",
lambda v: {"upper": v.upper(), "lower": v.lower()},
multi=True,
where="id > ?",
where_args=[1],
)
assert list(table.rows) == [
{"id": 1, "lower": None, "title": "One", "upper": None},
{"id": 2, "lower": "two", "title": "Two", "upper": "TWO"},
]
def test_convert_multi_exception(fresh_db):
table = fresh_db["table"]
table.insert({"title": "Mixed Case"})
with pytest.raises(BadMultiValues):
table.convert("title", lambda v: v.upper(), multi=True)

48
tests/test_create.py
View file

@ -1,7 +1,7 @@
from sqlite_utils.db import (
Index,
Database,
ForeignKey,
DescIndex,
AlterError,
NoObviousTable,
ForeignKey,
@ -147,8 +147,8 @@ def test_create_table_with_not_null(fresh_db):
)
def test_create_table_from_example(fresh_db, example, expected_columns):
people_table = fresh_db["people"]
assert None == people_table.last_rowid
assert None == people_table.last_pk
assert people_table.last_rowid is None
assert people_table.last_pk is None
people_table.insert(example)
assert 1 == people_table.last_rowid
assert 1 == people_table.last_pk
@ -514,7 +514,7 @@ def test_insert_row_alter_table(
def test_insert_row_alter_table_invalid_column_characters(fresh_db):
table = fresh_db["table"]
rowid = table.insert({"foo": "bar"}).last_pk
table.insert({"foo": "bar"}).last_pk
with pytest.raises(AssertionError):
table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True)
@ -739,6 +739,19 @@ def test_create_index_if_not_exists(fresh_db):
dogs.create_index(["name"], if_not_exists=True)
def test_create_index_desc(fresh_db):
dogs = fresh_db["dogs"]
dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True})
assert [] == dogs.indexes
dogs.create_index([DescIndex("age"), "name"])
sql = fresh_db.execute(
"select sql from sqlite_master where name='idx_dogs_age_name'"
).fetchone()[0]
assert sql == (
"CREATE INDEX [idx_dogs_age_name]\n" " ON [dogs] ([age] desc, [name])"
)
@pytest.mark.parametrize(
"data_structure",
(
@ -748,7 +761,7 @@ def test_create_index_if_not_exists(fresh_db):
{"dictionary": {"nested": "complex"}},
collections.OrderedDict(
[
("key1", {"nested": "complex"}),
("key1", {"nested": ["cømplex"]}),
("key2", "foo"),
]
),
@ -762,6 +775,14 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure):
assert data_structure == json.loads(row[1])
def test_insert_list_nested_unicode(fresh_db):
fresh_db["test"].insert(
{"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id"
)
row = fresh_db.execute("select id, data from test").fetchone()
assert row[1] == '{"key1": {"nested": ["cømplex"]}}'
def test_insert_uuid(fresh_db):
uuid4 = uuid.uuid4()
fresh_db["test"].insert({"uuid": uuid4})
@ -805,8 +826,8 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db
+ [{"i": 101, "extra": "Should trigger ALTER"}],
alter=True,
)
rows = fresh_db.execute_returning_dicts("select * from test where i = 101")
assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows
rows = list(fresh_db.query("select * from test where i = 101"))
assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}]
def test_insert_ignore(fresh_db):
@ -817,8 +838,8 @@ def test_insert_ignore(fresh_db):
# Using ignore=True should cause our insert to be silently ignored
fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True)
# Only one row, and it should be bar=2, not bar=3
rows = fresh_db.execute_returning_dicts("select * from test")
assert [{"id": 1, "bar": 2}] == rows
rows = list(fresh_db.query("select * from test"))
assert rows == [{"id": 1, "bar": 2}]
def test_insert_hash_id(fresh_db):
@ -848,8 +869,6 @@ def test_works_with_pathlib_path(tmpdir):
@pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed")
def test_create_table_numpy(fresh_db):
import numpy as np
df = pd.DataFrame({"col 1": range(3), "col 2": range(3)})
fresh_db["pandas"].insert_all(df.to_dict(orient="records"))
assert [
@ -969,6 +988,13 @@ def test_insert_all_empty_list(fresh_db):
assert 1 == fresh_db["t"].count
def test_insert_all_single_column(fresh_db):
table = fresh_db["table"]
table.insert_all([{"name": "Cleo"}], pk="name")
assert [{"name": "Cleo"}] == list(table.rows)
assert table.pks == ["name"]
def test_create_with_a_null_column(fresh_db):
record = {"name": "Name", "description": None}
fresh_db["t"].insert(record)

33
tests/test_docs.py
View file

@ -1,10 +1,12 @@
from sqlite_utils import cli
from click.testing import CliRunner
from sqlite_utils import cli, recipes
from pathlib import Path
import pytest
import re
docs_path = Path(__file__).parent.parent / "docs"
commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ")
recipes_re = re.compile(r"r\.(\w+)\(")
@pytest.fixture(scope="session")
@ -17,11 +19,36 @@ def documented_commands():
}
@pytest.fixture(scope="session")
def documented_recipes():
rst = (docs_path / "cli.rst").read_text()
return set(recipes_re.findall(rst))
@pytest.mark.parametrize("command", cli.cli.commands.keys())
def test_commands_are_documented(documented_commands, command):
assert command in documented_commands
@pytest.mark.parametrize("command", cli.cli.commands.values())
def test_commands_have_docstrings(command):
assert command.__doc__, "{} is missing a docstring".format(command)
def test_commands_have_help(command):
assert command.help, "{} is missing its help".format(command)
def test_convert_help():
result = CliRunner().invoke(cli.cli, ["convert", "--help"])
assert result.exit_code == 0
for expected in (
"r.jsonsplit(value, ",
"r.parsedate(value, ",
"r.parsedatetime(value, ",
):
assert expected in result.output
@pytest.mark.parametrize(
"recipe",
[n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")],
)
def test_recipes_are_documented(documented_recipes, recipe):
assert recipe in documented_recipes

29
tests/test_enable_counts.py
View file

@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db):
# Now enable counts
foo.enable_counts()
assert foo.triggers_dict == {
"foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) + 1\n );\nEND",
"foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) - 1\n );\nEND",
"foo_counts_insert": (
"CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n"
"BEGIN\n"
" INSERT OR REPLACE INTO [_counts]\n"
" VALUES (\n 'foo',\n"
" COALESCE(\n"
" (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
" 0\n"
" ) + 1\n"
" );\n"
"END"
),
"foo_counts_delete": (
"CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n"
"BEGIN\n"
" INSERT OR REPLACE INTO [_counts]\n"
" VALUES (\n"
" 'foo',\n"
" COALESCE(\n"
" (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n"
" 0\n"
" ) - 1\n"
" );\n"
"END"
),
}
assert fresh_db.table_names() == ["foo", "_counts"]
assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}]
@ -109,7 +132,7 @@ def test_uses_counts_after_enable_counts(counts_db_path):
assert db["foo"].count == 1
assert logged == [
("select name from sqlite_master where type = 'view'", None),
("select count(*) from [foo]", None),
("select count(*) from [foo]", []),
]
logged.clear()
assert not db.use_counts_table

17
tests/test_extract.py
View file

@ -1,4 +1,4 @@
from sqlite_utils.db import Index, InvalidColumns
from sqlite_utils.db import InvalidColumns
import itertools
import pytest
@ -126,12 +126,25 @@ def test_extract_rowid_table(fresh_db):
fresh_db["tree"].extract(["common_name", "latin_name"])
assert fresh_db["tree"].schema == (
'CREATE TABLE "tree" (\n'
" [rowid] INTEGER PRIMARY KEY,\n"
" [name] TEXT,\n"
" [common_name_latin_name_id] INTEGER,\n"
" FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
")"
)
assert (
fresh_db.execute(
"""
select
tree.name,
common_name_latin_name.common_name,
common_name_latin_name.latin_name
from tree
join common_name_latin_name
on tree.common_name_latin_name_id = common_name_latin_name.id
"""
).fetchall()
== [("Tree 1", "Palm", "Arecaceae")]
)
def test_reuse_lookup_table(fresh_db):

2
tests/test_extracts.py
View file

@ -1,4 +1,4 @@
from sqlite_utils.db import Index, ForeignKey
from sqlite_utils.db import Index
import pytest

1
tests/test_hypothesis.py
View file

@ -2,6 +2,7 @@ from hypothesis import given
import hypothesis.strategies as st
import sqlite_utils
# SQLite integers are -(2^63) to 2^63 - 1
@given(st.integers(-9223372036854775808, 9223372036854775807))
def test_roundtrip_integers(integer):

9
tests/test_insert_files.py
View file

@ -2,9 +2,11 @@ from sqlite_utils import cli, Database
from click.testing import CliRunner
import os
import pathlib
import pytest
def test_insert_files():
@pytest.mark.parametrize("silent", (False, True))
def test_insert_files(silent):
runner = CliRunner()
with runner.isolated_filesystem():
tmpdir = pathlib.Path(".")
@ -34,7 +36,10 @@ def test_insert_files():
cols += ["-c", "{}:{}".format(coltype, coltype)]
result = runner.invoke(
cli.cli,
["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"],
["insert-files", db_path, "files", str(tmpdir)]
+ cols
+ ["--pk", "path"]
+ (["--silent"] if silent else []),
catch_exceptions=False,
)
assert result.exit_code == 0, result.stdout

74
tests/test_introspect.py
View file

@ -1,4 +1,4 @@
from sqlite_utils.db import Index, View, Database
from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn
import pytest
@ -33,7 +33,7 @@ def test_detect_fts(existing_db):
assert "woo_fts" == existing_db["woo_fts"].detect_fts()
assert "woo2_fts" == existing_db["woo2"].detect_fts()
assert "woo2_fts" == existing_db["woo2_fts"].detect_fts()
assert None == existing_db["foo"].detect_fts()
assert existing_db["foo"].detect_fts() is None
def test_tables(existing_db):
@ -52,7 +52,14 @@ def test_views(fresh_db):
def test_count(existing_db):
assert 3 == existing_db["foo"].count
assert existing_db["foo"].count == 3
assert existing_db["foo"].count_where() == 3
assert existing_db["foo"].execute_count() == 3
def test_count_where(existing_db):
assert existing_db["foo"].count_where("text != ?", ["two"]) == 2
assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2
def test_columns(existing_db):
@ -62,8 +69,12 @@ def test_columns(existing_db):
]
def test_schema(existing_db):
assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema
def test_table_schema(existing_db):
assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)"
def test_database_schema(existing_db):
assert existing_db.schema == "CREATE TABLE foo (text TEXT);"
def test_table_repr(fresh_db):
@ -93,6 +104,33 @@ def test_indexes(fresh_db):
] == fresh_db["Gosh"].indexes
def test_xindexes(fresh_db):
fresh_db.executescript(
"""
create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_c1 on Gosh(c1);
create index Gosh_c2c3 on Gosh(c2, c3 desc);
"""
)
assert fresh_db["Gosh"].xindexes == [
XIndex(
name="Gosh_c2c3",
columns=[
XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1),
XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1),
XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0),
],
),
XIndex(
name="Gosh_c1",
columns=[
XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1),
XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0),
],
),
]
@pytest.mark.parametrize(
"column,expected_table_guess",
(
@ -144,9 +182,21 @@ def test_triggers_and_triggers_dict(fresh_db):
(t.name, t.table) for t in fresh_db["authors"].triggers
}
expected_triggers = {
"authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
"authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND",
"authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND",
"authors_ai": (
"CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n"
" INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n"
"END"
),
"authors_ad": (
"CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n"
" INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
"END"
),
"authors_au": (
"CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n"
" INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n"
" INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND"
),
}
assert authors.triggers_dict == expected_triggers
assert fresh_db["other"].triggers == []
@ -206,3 +256,11 @@ def test_virtual_table_using(sql, expected_name, expected_using):
db = Database(memory=True)
db.execute(sql)
assert db[expected_name].virtual_table_using == expected_using
def test_use_rowid():
db = Database(memory=True)
db["rowid_table"].insert({"name": "Cleo"})
db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id")
assert db["rowid_table"].use_rowid
assert not db["regular_table"].use_rowid

17
tests/test_query.py Normal file
View file

@ -0,0 +1,17 @@
import types
def test_query(fresh_db):
fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}])
results = fresh_db.query("select * from dogs order by name desc")
assert isinstance(results, types.GeneratorType)
assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}]
def test_execute_returning_dicts(fresh_db):
# Like db.query() but returns a list, included for backwards compatibility
# see https://github.com/simonw/sqlite-utils/issues/290
fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id")
assert fresh_db.execute_returning_dicts("select * from test") == [
{"id": 1, "bar": 2}
]

108
tests/test_recipes.py Normal file
View file

@ -0,0 +1,108 @@
from sqlite_utils import recipes
import json
import pytest
@pytest.fixture
def dates_db(fresh_db):
fresh_db["example"].insert_all(
[
{"id": 1, "dt": "5th October 2019 12:04"},
{"id": 2, "dt": "6th October 2019 00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
],
pk="id",
)
return fresh_db
def test_parsedate(dates_db):
dates_db["example"].convert("dt", recipes.parsedate)
assert list(dates_db["example"].rows) == [
{"id": 1, "dt": "2019-10-05"},
{"id": 2, "dt": "2019-10-06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
]
def test_parsedatetime(dates_db):
dates_db["example"].convert("dt", recipes.parsedatetime)
assert list(dates_db["example"].rows) == [
{"id": 1, "dt": "2019-10-05T12:04:00"},
{"id": 2, "dt": "2019-10-06T00:05:06"},
{"id": 3, "dt": ""},
{"id": 4, "dt": None},
]
@pytest.mark.parametrize(
"recipe,kwargs,expected",
(
("parsedate", {}, "2005-03-04"),
("parsedate", {"dayfirst": True}, "2005-04-03"),
("parsedatetime", {}, "2005-03-04T00:00:00"),
("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"),
),
)
def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected):
fresh_db["example"].insert_all(
[
{"id": 1, "dt": "03/04/05"},
],
pk="id",
)
fresh_db["example"].convert(
"dt", lambda value: getattr(recipes, recipe)(value, **kwargs)
)
assert list(fresh_db["example"].rows) == [
{"id": 1, "dt": expected},
]
@pytest.mark.parametrize("delimiter", [None, ";", "-"])
def test_jsonsplit(fresh_db, delimiter):
fresh_db["example"].insert_all(
[
{"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
{"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
],
pk="id",
)
fn = recipes.jsonsplit
if delimiter is not None:
def fn(value):
return recipes.jsonsplit(value, delimiter=delimiter)
fresh_db["example"].convert("tags", fn)
assert list(fresh_db["example"].rows) == [
{"id": 1, "tags": '["foo", "bar"]'},
{"id": 2, "tags": '["bar", "baz"]'},
]
@pytest.mark.parametrize(
"type,expected",
(
(None, ["1", "2", "3"]),
(float, [1.0, 2.0, 3.0]),
(int, [1, 2, 3]),
),
)
def test_jsonsplit_type(fresh_db, type, expected):
fresh_db["example"].insert_all(
[
{"id": 1, "records": "1,2,3"},
],
pk="id",
)
fn = recipes.jsonsplit
if type is not None:
def fn(value):
return recipes.jsonsplit(value, type=type)
fresh_db["example"].convert("records", fn)
assert json.loads(fresh_db["example"].get(1)["records"]) == expected

2
tests/test_recreate.py
View file

@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory():
def test_recreate_not_allowed_for_connection():
conn = sqlite3.connect(":memory:")
with pytest.raises(AssertionError):
db = Database(conn, recreate=True)
Database(conn, recreate=True)
@pytest.mark.parametrize(

5
tests/test_register_function.py
View file

@ -1,3 +1,4 @@
# flake8: noqa
import pytest
import sys
from unittest.mock import MagicMock
@ -55,14 +56,14 @@ def test_register_function_replace(fresh_db):
# This will fail to replace the function:
@fresh_db.register_function()
def one():
def one(): # noqa
return "two"
assert "one" == fresh_db.execute("select one()").fetchone()[0]
# This will replace it
@fresh_db.register_function(replace=True)
def one():
def one(): # noqa
return "two"
assert "two" == fresh_db.execute("select one()").fetchone()[0]

2
tests/test_rows.py
View file

@ -1,4 +1,3 @@
from sqlite_utils.db import Index, View
import pytest
@ -13,6 +12,7 @@ def test_rows(existing_db):
[
("name = ?", ["Pancakes"], {2}),
("age > ?", [3], {1}),
("age > :age", {"age": 3}, {1}),
("name is not null", [], {1, 2}),
("is_good = ?", [True], {1, 2}),
],

35
tests/test_tracer.py
View file

@ -1,4 +1,3 @@
import pytest
from sqlite_utils import Database
@ -32,7 +31,9 @@ def test_tracer():
def test_with_tracer():
collected = []
tracer = lambda sql, params: collected.append((sql, params))
def tracer(sql, params):
return collected.append((sql, params))
db = Database(memory=True)
@ -48,13 +49,39 @@ def test_with_tracer():
assert collected == [
("select name from sqlite_master where type = 'view'", None),
(
"SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )",
(
"SELECT name FROM sqlite_master\n"
" WHERE rootpage = 0\n"
" AND (\n"
" sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n"
" OR (\n"
' tbl_name = "dogs"\n'
" AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n"
" )\n"
" )"
),
None,
),
("select name from sqlite_master where type = 'view'", None),
("select sql from sqlite_master where name = ?", ("dogs_fts",)),
(
"with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n [original].*\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n [dogs_fts].rank",
(
"with original as (\n"
" select\n"
" rowid,\n"
" *\n"
" from [dogs]\n"
")\n"
"select\n"
" [original].*\n"
"from\n"
" [original]\n"
" join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n"
"where\n"
" [dogs_fts] match :query\n"
"order by\n"
" [dogs_fts].rank"
),
{"query": "Cleopaws"},
),
]

84
tests/test_transform.py
View file

@ -89,9 +89,14 @@ import pytest
],
)
@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys):
def test_transform_sql_table_with_primary_key(
fresh_db, params, expected_sql, use_pragma_foreign_keys
):
captured = []
tracer = lambda sql, params: captured.append((sql, params))
def tracer(sql, params):
return captured.append((sql, params))
dogs = fresh_db["dogs"]
if use_pragma_foreign_keys:
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
@ -111,7 +116,80 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys):
assert ("PRAGMA foreign_keys=1;", None) not in captured
def test_transform_sql_rowid_to_id(fresh_db):
@pytest.mark.parametrize(
"params,expected_sql",
[
# Identity transform - nothing changes
(
{},
[
"CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);",
"INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];",
"DROP TABLE [dogs];",
"ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
],
),
# Change column type
(
{"types": {"age": int}},
[
"CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] INTEGER\n);",
"INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];",
"DROP TABLE [dogs];",
"ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
],
),
# Rename a column
(
{"rename": {"age": "dog_age"}},
[
"CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [dog_age] TEXT\n);",
"INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];",
"DROP TABLE [dogs];",
"ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
],
),
# Make ID a primary key
(
{"pk": "id"},
[
"CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);",
"INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];",
"DROP TABLE [dogs];",
"ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];",
],
),
],
)
@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
def test_transform_sql_table_with_no_primary_key(
fresh_db, params, expected_sql, use_pragma_foreign_keys
):
captured = []
def tracer(sql, params):
return captured.append((sql, params))
dogs = fresh_db["dogs"]
if use_pragma_foreign_keys:
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}})
assert sql == expected_sql
# Check that .transform() runs without exceptions:
with fresh_db.tracer(tracer):
dogs.transform(**params)
# If use_pragma_foreign_keys, check that we did the right thing
if use_pragma_foreign_keys:
assert ("PRAGMA foreign_keys=0;", None) in captured
assert captured[-2] == ("PRAGMA foreign_key_check;", None)
assert captured[-1] == ("PRAGMA foreign_keys=1;", None)
else:
assert ("PRAGMA foreign_keys=0;", None) not in captured
assert ("PRAGMA foreign_keys=1;", None) not in captured
def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db):
dogs = fresh_db["dogs"]
dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
assert (

9
tests/test_upsert.py
View file

@ -21,6 +21,13 @@ def test_upsert_all(fresh_db):
assert table.last_pk is None
def test_upsert_all_single_column(fresh_db):
table = fresh_db["table"]
table.upsert_all([{"name": "Cleo"}], pk="name")
assert [{"name": "Cleo"}] == list(table.rows)
assert table.pks == ["name"]
def test_upsert_error_if_no_pk(fresh_db):
table = fresh_db["table"]
with pytest.raises(PrimaryKeyRequired):
@ -47,7 +54,7 @@ def test_upsert_compound_primary_key(fresh_db):
],
pk=("species", "id"),
)
assert None == table.last_pk
assert table.last_pk is None
table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id"))
assert ("dog", 1) == table.last_pk
assert [

1
tests/test_wal.py
View file

@ -1,6 +1,5 @@
import pytest
from sqlite_utils import Database
import sqlite3
@pytest.fixture