diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9f05567..c0bd779 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 0000000..8a86cd2 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,27 @@ +name: Check spelling in documentation + +on: [push, pull_request] + +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[docs]' + - name: Check spelling + run: | + codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml new file mode 100644 index 0000000..4b99cc5 --- /dev/null +++ b/.github/workflows/test-coverage.yml @@ -0,0 +1,41 @@ +name: Calculate test coverage + +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e .[test] + python -m pip install pytest-cov + - name: Run tests + run: |- + ls -lah + pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term + ls -lah + - name: Upload coverage report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8a79ae..5dc3fe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,12 +25,16 @@ jobs: ${{ runner.os }}-pip- - name: Install dependencies run: | - pip install -e '.[test]' + pip install -e '.[test,mypy,flake8]' - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy - name: Run tests run: | pytest + - name: run mypy + run: mypy sqlite_utils + - name: run flake8 + run: flake8 - name: Check formatting run: black . --check diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..ce66cbe --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +version: 2 + +sphinx: + configuration: docs/conf.py + +python: + version: "3.8" + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/README.md b/README.md index e8b8245..bac6369 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,11 @@ # sqlite-utils [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) -[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/latest/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) -[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=latest)](http://sqlite-utils.datasette.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) +[![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) Python CLI utility and library for manipulating SQLite databases. @@ -56,7 +57,11 @@ You can import JSON data into a new database table like this: Or for data in a CSV file: - $ sqlite-utils insert dogs.db dogs docs.csv --csv + $ sqlite-utils insert dogs.db dogs dogs.csv --csv + +`sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: + + $ cat dogs.csv | sqlite-utils memory - "select name, age from dogs" See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..bfdc987 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true diff --git a/docs/Makefile b/docs/Makefile index a279768..5578ae3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -20,4 +20,4 @@ help: @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) livehtml: - sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) + sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d4770f..ab424ed 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,184 @@ Changelog =========== +.. _v3_15.1: + +3.15.1 (2021-08-10) +------------------- + +- Python library now includes type annotations on almost all of the methods, plus detailed docstrings describing each one. (:issue:`311`) +- New :ref:`reference` documentation page, powered by those docstrings. +- Fixed bug where ``.add_foreign_keys()`` failed to raise an error if called against a ``View``. (:issue:`313`) +- Fixed bug where ``.delete_where()`` returned a ``[]`` instead of returning ``self`` if called against a non-existent table. (:issue:`315`) + +.. _v3_15: + +3.15 (2021-08-09) +----------------- + +- ``sqlite-utils insert --flatten`` option for :ref:`flattening nested JSON objects ` to create tables with column names like ``topkey_nestedkey``. (:issue:`310`) +- Fixed several spelling mistakes in the documentation, spotted `using codespell `__. +- Errors that occur while using the ``sqlite-utils`` CLI tool now show the responsible SQL and query parameters, if possible. (:issue:`309`) + +.. _v3_14: + +3.14 (2021-08-02) +----------------- + +This release introduces the new :ref:`sqlite-utils convert command ` (:issue:`251`) and corresponding :ref:`table.convert(...) ` Python method (:issue:`302`). These tools can be used to apply a Python conversion function to one or more columns of a table, either updating the column in place or using transformed data from that column to populate one or more other columns. + +This command-line example uses the Python standard library `textwrap module `__ to wrap the content of the ``content`` column in the ``articles`` table to 100 characters:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 100))' \ + --import=textwrap + +The same operation in Python code looks like this: + +.. code-block:: python + + import sqlite_utils, textwrap + + db = sqlite_utils.Database("content.db") + db["articles"].convert("content", lambda v: "\n".join(textwrap.wrap(v, 100))) + +See the full documentation for the :ref:`sqlite-utils convert command ` and the :ref:`table.convert(...) ` Python method for more details. + +Also in this release: + +- The new ``table.count_where(...)`` method, for counting rows in a table that match a specific SQL ``WHERE`` clause. (:issue:`305`) +- New ``--silent`` option for the :ref:`sqlite-utils insert-files command ` to hide the terminal progress bar, consistent with the ``--silent`` option for ``sqlite-utils convert``. (:issue:`301`) + +.. _v3_13: + +3.13 (2021-07-24) +----------------- + +- ``sqlite-utils schema my.db table1 table2`` command now accepts optional table names. (:issue:`299`) +- ``sqlite-utils memory --help`` now describes the ``--schema`` option. + +.. _v3_12: + +3.12 (2021-06-25) +----------------- + +- New :ref:`db.query(sql, params) ` method, which executes a SQL query and returns the results as an iterator over Python dictionaries. (:issue:`290`) +- This project now uses ``flake8`` and has started to use ``mypy``. (:issue:`291`) +- New documentation on :ref:`contributing ` to this project. (:issue:`292`) + +.. _v3_11: + +3.11 (2021-06-20) +----------------- + +- New ``sqlite-utils memory data.csv --schema`` option, for outputting the schema of the in-memory database generated from one or more files. See :ref:`cli_memory_schema_dump_save`. (:issue:`288`) +- Added :ref:`installation instructions `. (:issue:`286`) + +.. _v3_10: + +3.10 (2021-06-19) +----------------- + +This release introduces the ``sqlite-utils memory`` command, which can be used to load CSV or JSON data into a temporary in-memory database and run SQL queries (including joins across multiple files) directly against that data. + +Also new: ``sqlite-utils insert --detect-types``, ``sqlite-utils dump``, ``table.use_rowid`` plus some smaller fixes. + +sqlite-utils memory +~~~~~~~~~~~~~~~~~~~ + +This example of ``sqlite-utils memory`` retrieves information about the all of the repositories in the `Dogsheep `__ organization on GitHub using `this JSON API `__, sorts them by their number of stars and outputs a table of the top five (using ``-t``):: + + $ curl -s 'https://api.github.com/users/dogsheep/repos' \ + | sqlite-utils memory - ' + select full_name, forks_count, stargazers_count + from stdin order by stargazers_count desc limit 5 + ' -t + full_name forks_count stargazers_count + --------------------------------- ------------- ------------------ + dogsheep/twitter-to-sqlite 12 225 + dogsheep/github-to-sqlite 14 139 + dogsheep/dogsheep-photos 5 116 + dogsheep/dogsheep.github.io 7 90 + dogsheep/healthkit-to-sqlite 4 85 + +The tool works against files on disk as well. This example joins data from two CSV files:: + + $ cat creatures.csv + species_id,name + 1,Cleo + 2,Bants + 2,Dori + 2,Azi + $ cat species.csv + id,species_name + 1,Dog + 2,Chicken + $ sqlite-utils memory species.csv creatures.csv ' + select * from creatures join species on creatures.species_id = species.id + ' + [{"species_id": 1, "name": "Cleo", "id": 1, "species_name": "Dog"}, + {"species_id": 2, "name": "Bants", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Dori", "id": 2, "species_name": "Chicken"}, + {"species_id": 2, "name": "Azi", "id": 2, "species_name": "Chicken"}] + +Here the ``species.csv`` file becomes the ``species`` table, the ``creatures.csv`` file becomes the ``creatures`` table and the output is JSON, the default output format. + +You can also use the ``--attach`` option to attach existing SQLite database files to the in-memory database, in order to join data from CSV or JSON directly against your existing tables. + +Full documentation of this new feature is available in :ref:`cli_memory`. (:issue:`272`) + +sqlite-utils insert \-\-detect-types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`sqlite-utils insert ` command can be used to insert data from JSON, CSV or TSV files into a SQLite database file. The new ``--detect-types`` option (shortcut ``-d``), when used in conjunction with a CSV or TSV import, will automatically detect if columns in the file are integers or floating point numbers as opposed to treating everything as a text column and create the new table with the corresponding schema. See :ref:`cli_insert_csv_tsv` for details. (:issue:`282`) + +Other changes +~~~~~~~~~~~~~ + +- **Bug fix**: ``table.transform()``, when run against a table without explicit primary keys, would incorrectly create a new version of the table with an explicit primary key column called ``rowid``. (:issue:`284`) +- New ``table.use_rowid`` introspection property, see :ref:`python_api_introspection_use_rowid`. (:issue:`285`) +- The new ``sqlite-utils dump file.db`` command outputs a SQL dump that can be used to recreate a database. (:issue:`274`) +- ``-h`` now works as a shortcut for ``--help``, thanks Loren McIntyre. (:issue:`276`) +- Now using `pytest-cov `__ and `Codecov `__ to track test coverage - currently at 96%. (:issue:`275`) +- SQL errors that occur when using ``sqlite-utils query`` are now displayed as CLI errors. + +.. _v3_9_1: + +3.9.1 (2021-06-12) +------------------ + +- Fixed bug when using ``table.upsert_all()`` to create a table with only a single column that is treated as the primary key. (:issue:`271`) + +.. _v3_9: + +3.9 (2021-06-11) +---------------- + +- New ``sqlite-utils schema`` command showing the full SQL schema for a database, see :ref:`Showing the schema (CLI)`. (:issue:`268`) +- ``db.schema`` introspection property exposing the same feature to the Python library, see :ref:`Showing the schema (Python library) `. + +.. _v3_8: + +3.8 (2021-06-02) +---------------- + +- New ``sqlite-utils indexes`` command to list indexes in a database, see :ref:`cli_indexes`. (:issue:`263`) +- ``table.xindexes`` introspection property returning more details about that table's indexes, see :ref:`python_api_introspection_xindexes`. (:issue:`261`) + +.. _v3_7: + +3.7 (2021-05-28) +---------------- + +- New ``table.pks_and_rows_where()`` method returning ``(primary_key, row_dictionary)`` tuples - see :ref:`python_api_pks_and_rows_where`. (:issue:`240`) +- Fixed bug with ``table.add_foreign_key()`` against columns containing spaces. (:issue:`238`) +- ``table_or_view.drop(ignore=True)`` option for avoiding errors if the table or view does not exist. (:issue:`237`) +- ``sqlite-utils drop-view --ignore`` and ``sqlite-utils drop-table --ignore`` options. (:issue:`237`) +- Fixed a bug with inserts of nested JSON containing non-ascii strings - thanks, Dylan Wu. (:issue:`257`) +- Suggest ``--alter`` if an error occurs caused by a missing column. (:issue:`259`) +- Support creating indexes with columns in descending order, see :ref:`API documentation ` and :ref:`CLI documentation `. (:issue:`260`) +- Correctly handle CSV files that start with a UTF-8 BOM. (:issue:`250`) + .. _v3_6: 3.6 (2021-02-18) @@ -9,73 +187,73 @@ This release adds the ability to execute queries joining data from more than one database file - similar to the cross database querying feature introduced in `Datasette 0.55 `__. -- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (`#113 `__) -- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (`#236 `__) +- The ``db.attach(alias, filepath)`` Python method can be used to attach extra databases to the same connection, see :ref:`db.attach() in the Python API documentation `. (:issue:`113`) +- The ``--attach`` option attaches extra aliased databases to run SQL queries against directly on the command-line, see :ref:`attaching additional databases in the CLI documentation `. (:issue:`236`) .. _v3_5: 3.5 (2021-02-14) ---------------- -- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (`#230 `__) -- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (`#231 `__) -- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (`#228 `__) -- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (`#234 `__) -- Fixed bug importing CSV files with columns containing more than 128KB of data. (`#229 `__) -- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (`#232 `__) +- ``sqlite-utils insert --sniff`` option for detecting the delimiter and quote character used by a CSV file, see :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`230`) +- The ``table.rows_where()``, ``table.search()`` and ``table.search_sql()`` methods all now take optional ``offset=`` and ``limit=`` arguments. (:issue:`231`) +- New ``--no-headers`` option for ``sqlite-utils insert --csv`` to handle CSV files that are missing the header row, see :ref:`cli_insert_csv_tsv_no_header`. (:issue:`228`) +- Fixed bug where inserting data with extra columns in subsequent chunks would throw an error. Thanks `@nieuwenhoven `__ for the fix. (:issue:`234`) +- Fixed bug importing CSV files with columns containing more than 128KB of data. (:issue:`229`) +- Test suite now runs in CI against Ubuntu, macOS and Windows. Thanks `@nieuwenhoven `__ for the Windows test fixes. (:issue:`232`) .. _v3_4_1: 3.4.1 (2021-02-05) ------------------ -- Fixed a code import bug that slipped in to 3.4. (`#226 `__) +- Fixed a code import bug that slipped in to 3.4. (:issue:`226`) .. _v3_4: 3.4 (2021-02-05) ---------------- -- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (`#223 `__) +- ``sqlite-utils insert --csv`` now accepts optional ``--delimiter`` and ``--quotechar`` options. See :ref:`cli_insert_csv_tsv_delimiter`. (:issue:`223`) .. _v3_3: 3.3 (2021-01-17) ---------------- -- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (`#222 `__) +- The ``table.m2m()`` method now accepts an optional ``alter=True`` argument to specify that any missing columns should be added to the referenced table. See :ref:`python_api_m2m`. (:issue:`222`) .. _v3_2_1: 3.2.1 (2021-01-12) ------------------ -- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (`#221 `__) +- Fixed a bug where ``.add_missing_columns()`` failed to take case insensitive column names into account. (:issue:`221`) .. _v3_2: 3.2 (2021-01-03) ---------------- -This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (`#212 `__) +This release introduces a new mechanism for speeding up ``count(*)`` queries using cached table counts, stored in a ``_counts`` table and updated by triggers. This mechanism is described in :ref:`python_api_cached_table_counts`, and can be enabled using Python API methods or the new ``enable-counts`` CLI command. (:issue:`212`) - ``table.enable_counts()`` method for enabling these triggers on a specific table. -- ``db.enable_counts()`` method for enabling triggers on every table in the database. (`#213 `__) -- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (`#214 `__) -- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (`#218 `__) -- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (`#215 `__) +- ``db.enable_counts()`` method for enabling triggers on every table in the database. (:issue:`213`) +- New ``sqlite-utils enable-counts my.db`` command for enabling counts on all or specific tables, see :ref:`cli_enable_counts`. (:issue:`214`) +- New ``sqlite-utils triggers`` command for listing the triggers defined for a database or specific tables, see :ref:`cli_triggers`. (:issue:`218`) +- New ``db.use_counts_table`` property which, if ``True``, causes ``table.count`` to read from the ``_counts`` table. (:issue:`215`) - ``table.has_counts_triggers`` property revealing if a table has been configured with the new ``_counts`` database triggers. -- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (`#219 `__) -- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (`#217 `__) -- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (`#211 `__, `#216 `__) -- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (`#206 `__) +- ``db.reset_counts()`` method and ``sqlite-utils reset-counts`` command for resetting the values in the ``_counts`` table. (:issue:`219`) +- The previously undocumented ``db.escape()`` method has been renamed to ``db.quote()`` and is now covered by the documentation: :ref:`python_api_quote`. (:issue:`217`) +- New ``table.triggers_dict`` and ``db.triggers_dict`` introspection properties. (:issue:`211`, :issue:`216`) +- ``sqlite-utils insert`` now shows a more useful error message for invalid JSON. (:issue:`206`) .. _v3_1_1: 3.1.1 (2021-01-01) ------------------ -- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (`#209 `__) +- Fixed failing test caused by ``optimize`` sometimes creating larger database files. (:issue:`209`) - Documentation now lives on https://sqlite-utils.datasette.io/ - README now includes ``brew install sqlite-utils`` installation method. @@ -84,7 +262,7 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.1 (2020-12-12) ---------------- -- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (`#207 `__) +- New command: ``sqlite-utils analyze-tables my.db`` outputs useful information about the table columns in the database, such as the number of distinct values and how many rows are null. See :ref:`cli_analyze_tables` for documentation. (:issue:`207`) - New ``table.analyze_column(column)`` Python method used by the ``analyze-tables`` command - see :ref:`python_api_analyze_column`. - The ``table.update()`` method now correctly handles values that should be stored as JSON. Thanks, Andreas Madsack. (`#204 `__) @@ -93,28 +271,28 @@ This release introduces a new mechanism for speeding up ``count(*)`` queries usi 3.0 (2020-11-08) ---------------- -This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (`#192 `__) +This release introduces a new ``sqlite-utils search`` command for searching tables, see :ref:`cli_search`. (:issue:`192`) -The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (`#197 `__) +The ``table.search()`` method has been redesigned, see :ref:`python_api_fts_search`. (:issue:`197`) The release includes minor backwards-incompatible changes, hence the version bump to 3.0. Those changes, which should not affect most users, are: - The ``-c`` shortcut option for outputting CSV is no longer available. The full ``--csv`` option is required instead. - The ``-f`` shortcut for ``--fmt`` has also been removed - use ``--fmt``. -- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (`#198 `__) +- The ``table.search()`` method now defaults to sorting by relevance, not sorting by ``rowid``. (:issue:`198`) - The ``table.search()`` method now returns a generator over a list of Python dictionaries. It previously returned a list of tuples. Also in this release: -- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (`#193 `__) -- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (`#196 `__) +- The ``query``, ``tables``, ``rows`` and ``search`` CLI commands now accept a new ``--tsv`` option which outputs the results in TSV. (:issue:`193`) +- A new ``table.virtual_table_using`` property reveals if a table is a virtual table, and returns the upper case type of virtual table (e.g. ``FTS4`` or ``FTS5``) if it is. It returns ``None`` if the table is not a virtual table. (:issue:`196`) - The new ``table.search_sql()`` method returns the SQL for searching a table, see :ref:`python_api_fts_search_sql`. -- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (`#200 `__) +- ``sqlite-utils rows`` now accepts multiple optional ``-c`` parameters specifying the columns to return. (:issue:`200`) Changes since the 3.0a0 alpha release: - The ``sqlite-utils search`` command now defaults to returning every result, unless you add a ``--limit 20`` option. -- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (`#201 `__) +- The ``sqlite-utils search -c`` and ``table.search(columns=[])`` options are now fully respected. (:issue:`201`) .. _v2_23: @@ -122,30 +300,30 @@ Changes since the 3.0a0 alpha release: ----------------- - ``table.m2m(other_table, records)`` method now takes any iterable, not just a list or tuple. Thanks, Adam Wolf. (`#189 `__) -- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (`#173 `__) -- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (`#191 `__) +- ``sqlite-utils insert`` now displays a progress bar for CSV or TSV imports. (:issue:`173`) +- New ``@db.register_function(deterministic=True)`` option for registering deterministic SQLite functions in Python 3.8 or higher. (:issue:`191`) .. _v2_22: 2.22 (2020-10-16) ----------------- -- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (`#182 `__) -- The ``--load-extension`` option is now available to many more commands. (`#137 `__) -- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (`#136 `__) -- Tests now also run against Python 3.9. (`#184 `__) -- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (`#181 `__) +- New ``--encoding`` option for processing CSV and TSV files that use a non-utf-8 encoding, for both the ``insert`` and ``update`` commands. (:issue:`182`) +- The ``--load-extension`` option is now available to many more commands. (:issue:`137`) +- ``--load-extension=spatialite`` can be used to load SpatiaLite from common installation locations, if it is available. (:issue:`136`) +- Tests now also run against Python 3.9. (:issue:`184`) +- Passing ``pk=["id"]`` now has the same effect as passing ``pk="id"``. (:issue:`181`) .. _v2_21: 2.21 (2020-09-24) ----------------- -- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (`#172 `__) +- ``table.extract()`` and ``sqlite-utils extract`` now apply much, much faster - one example operation reduced from twelve minutes to just four seconds! (:issue:`172`) - ``sqlite-utils extract`` no longer shows a progress bar, because it's fast enough not to need one. -- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (`#175 `__) -- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (`#176 `__) -- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (`#177 `__) +- New ``column_order=`` option for ``table.transform()`` which can be used to alter the order of columns in a table. (:issue:`175`) +- ``sqlite-utils transform --column-order=`` option (with a ``-o`` shortcut) for changing column order. (:issue:`176`) +- The ``table.transform(drop_foreign_keys=)`` parameter and the ``sqlite-utils transform --drop-foreign-key`` option have changed. They now accept just the name of the column rather than requiring all three of the column, other table and other column. This is technically a backwards-incompatible change but I chose not to bump the major version number because the transform feature is so new. (:issue:`177`) - The table ``.disable_fts()``, ``.rebuild_fts()``, ``.delete()``, ``.delete_where()`` and ``.add_missing_columns()`` methods all now ``return self``, which means they can be chained together with other table operations. .. _v2_20: @@ -153,7 +331,7 @@ Changes since the 3.0a0 alpha release: 2.20 (2020-09-22) ----------------- -This release introduces two key new capabilities: **transform** (`#114 `__) and **extract** (`#42 `__). +This release introduces two key new capabilities: **transform** (:issue:`114`) and **extract** (:issue:`42`). Transform ~~~~~~~~~ @@ -174,7 +352,7 @@ The Python library :ref:`extract() documentation ` describes Other changes ~~~~~~~~~~~~~ -- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (`#162 `__) +- The ``@db.register_function`` decorator can be used to quickly register Python functions as custom SQL functions, see :ref:`python_api_register_function`. (:issue:`162`) - The ``table.rows_where()`` method now accepts an optional ``select=`` argument for specifying which columns should be selected, see :ref:`python_api_rows`. .. _v2_19: @@ -182,31 +360,31 @@ Other changes 2.19 (2020-09-20) ----------------- -- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (`#157 `__) -- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (`#160 `__) -- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (`#112 `__) +- New ``sqlite-utils add-foreign-keys`` command for :ref:`cli_add_foreign_keys`. (:issue:`157`) +- New ``table.enable_fts(..., replace=True)`` argument for replacing an existing FTS table with a new configuration. (:issue:`160`) +- New ``table.add_foreign_key(..., ignore=True)`` argument for ignoring a foreign key if it already exists. (:issue:`112`) .. _v2_18: 2.18 (2020-09-08) ----------------- -- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (`#155 `__) -- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (`#155 `__) +- ``table.rebuild_fts()`` method for rebuilding a FTS index, see :ref:`python_api_fts_rebuild`. (:issue:`155`) +- ``sqlite-utils rebuild-fts data.db`` command for rebuilding FTS indexes across all tables, or just specific tables. (:issue:`155`) - ``table.optimize()`` method no longer deletes junk rows from the ``*_fts_docsize`` table. This was added in 2.17 but it turns out running ``table.rebuild_fts()`` is a better solution to this problem. -- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (`#145 `__) +- Fixed a bug where rows with additional columns that are inserted after the first batch of records could cause an error due to breaking SQLite's maximum number of parameters. Thanks, Simon Wiles. (:issue:`145`) .. _v2_17: 2.17 (2020-09-07) ----------------- -This release handles a bug where replacing rows in FTS tables could result in growing numbers of unneccessary rows in the associated ``*_fts_docsize`` table. (`#149 `__) +This release handles a bug where replacing rows in FTS tables could result in growing numbers of unnecessary rows in the associated ``*_fts_docsize`` table. (:issue:`149`) -- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (`#152 `__) -- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (`#153 `__) -- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (`#150 `__) -- Neater indentation for schema SQL. (`#148 `__) +- ``PRAGMA recursive_triggers=on`` by default for all connections. You can turn it off with ``Database(recursive_triggers=False)``. (:issue:`152`) +- ``table.optimize()`` method now deletes unnecessary rows from the ``*_fts_docsize`` table. (:issue:`153`) +- New tracer method for tracking underlying SQL queries, see :ref:`python_api_tracing`. (:issue:`150`) +- Neater indentation for schema SQL. (:issue:`148`) - Documentation for ``sqlite_utils.AlterError`` exception thrown by in ``add_foreign_keys()``. .. _v2_16_1: @@ -214,23 +392,23 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.16.1 (2020-08-28) ------------------- -- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (`#139 `__) -- Continuous Integration is now powered by GitHub Actions. (`#143 `__) +- ``insert_all(..., alter=True)`` now works for columns introduced after the first 100 records. Thanks, Simon Wiles! (:issue:`139`) +- Continuous Integration is now powered by GitHub Actions. (:issue:`143`) .. _v2_16: 2.16 (2020-08-21) ----------------- -- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (`#134 `__) -- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (`#135 `__) +- ``--load-extension`` option for ``sqlite-utils query`` for loading SQLite extensions. (:issue:`134`) +- New ``sqlite_utils.utils.find_spatialite()`` function for finding SpatiaLite in common locations. (:issue:`135`) .. _v2_15_1: 2.15.1 (2020-08-12) ------------------- -- Now available as a ``sdist`` package on PyPI in addition to a wheel. (`#133 `__) +- Now available as a ``sdist`` package on PyPI in addition to a wheel. (:issue:`133`) .. _v2_15: @@ -238,7 +416,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr ----------------- - New ``db.enable_wal()`` and ``db.disable_wal()`` methods for enabling and disabling `Write-Ahead Logging `__ for a database file - see :ref:`python_api_wal` in the Python API documentation. -- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (`#132 `__) +- Also ``sqlite-utils enable-wal file.db`` and ``sqlite-utils disable-wal file.db`` commands for doing the same thing on the command-line, see :ref:`WAL mode (CLI) `. (:issue:`132`) .. _v2_14_1: @@ -252,8 +430,8 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.14 (2020-08-01) ----------------- -- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (`#127 `__) -- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (`#130 `__) +- The :ref:`insert-files command ` can now read from standard input: ``cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg``. (:issue:`127`) +- You can now specify a full-text search tokenizer using the new ``tokenize=`` parameter to :ref:`enable_fts() `. This means you can enable Porter stemming on a table by running ``db["articles"].enable_fts(["headline", "body"], tokenize="porter")``. (:issue:`130`) - You can also set a custom tokenizer using the :ref:`sqlite-utils enable-fts ` CLI command, via the new ``--tokenize`` option. .. _v2_13: @@ -261,7 +439,7 @@ This release handles a bug where replacing rows in FTS tables could result in gr 2.13 (2020-07-29) ----------------- -- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (`#128 `__) +- ``memoryview`` and ``uuid.UUID`` objects are now supported. ``memoryview`` objects will be stored using ``BLOB`` and ``uuid.UUID`` objects will be stored using ``TEXT``. (:issue:`128`) .. _v2_12: @@ -270,11 +448,11 @@ This release handles a bug where replacing rows in FTS tables could result in gr The theme of this release is better tools for working with binary data. The new ``insert-files`` command can be used to insert binary files directly into a database table, and other commands have been improved with better support for BLOB columns. -- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (`#122 `__) -- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (`#123 `__) -- JSON output now encodes BLOB values as special base64 obects - see :ref:`cli_query_json`. (`#125 `__) -- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (`#126 `__) -- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (`#124 `__) +- ``sqlite-utils insert-files my.db gifs *.gif`` can now insert the contents of files into a specified table. The columns in the table can be customized to include different pieces of metadata derived from the files. See :ref:`cli_insert_files`. (:issue:`122`) +- ``--raw`` option to ``sqlite-utils query`` - for outputting just a single raw column value - see :ref:`cli_query_raw`. (:issue:`123`) +- JSON output now encodes BLOB values as special base64 objects - see :ref:`cli_query_json`. (:issue:`125`) +- The same format of JSON base64 objects can now be used to insert binary data - see :ref:`cli_inserting_data`. (:issue:`126`) +- The ``sqlite-utils query`` command can now accept named parameters, e.g. ``sqlite-utils :memory: "select :num * :num2" -p num 5 -p num2 6`` - see :ref:`cli_query_json`. (:issue:`124`) .. _v2_11: @@ -289,14 +467,14 @@ The theme of this release is better tools for working with binary data. The new 2.10.1 (2020-06-23) ------------------- -- Added documentation for the ``table.pks`` introspection property. (`#116 `__) +- Added documentation for the ``table.pks`` introspection property. (:issue:`116`) .. _v2_10: 2.10 (2020-06-12) ----------------- -- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (`#115 `__) +- The ``sqlite-utils`` command now supports UPDATE/INSERT/DELETE in addition to SELECT. (:issue:`115`) .. _v2_9_1: @@ -310,77 +488,77 @@ The theme of this release is better tools for working with binary data. The new 2.9 (2020-05-10) ---------------- -- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (`#111 `__) +- New ``sqlite-utils drop-table`` command, see :ref:`cli_drop_table`. (:issue:`111`) - New ``sqlite-utils drop-view`` command, see :ref:`cli_drop_view`. -- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (`#110 `__) +- Python ``decimal.Decimal`` objects are now stored as ``FLOAT``. (:issue:`110`) .. _v2_8: 2.8 (2020-05-03) ---------------- -- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (`#27 `__) -- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (`#107 `__) +- New ``sqlite-utils create-table`` command, see :ref:`cli_create_table`. (:issue:`27`) +- New ``sqlite-utils create-view`` command, see :ref:`cli_create_view`. (:issue:`107`) .. _v2_7.2: 2.7.2 (2020-05-02) ------------------ -- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (`#106 `__) +- ``db.create_view(...)`` now has additional parameters ``ignore=True`` or ``replace=True``, see :ref:`python_api_create_view`. (:issue:`106`) .. _v2_7.1: 2.7.1 (2020-05-01) ------------------ -- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (`#105 `__) -- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (`#104 `__) -- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (`#102 `__) +- New ``sqlite-utils views my.db`` command for listing views in a database, see :ref:`cli_views`. (:issue:`105`) +- ``sqlite-utils tables`` (and ``views``) has a new ``--schema`` option which outputs the table/view schema, see :ref:`cli_tables`. (:issue:`104`) +- Nested structures containing invalid JSON values (e.g. Python bytestrings) are now serialized using ``repr()`` instead of throwing an error. (:issue:`102`) .. _v2_7: 2.7 (2020-04-17) ---------------- -- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (`#100 `__) +- New ``columns=`` argument for the ``.insert()``, ``.insert_all()``, ``.upsert()`` and ``.upsert_all()`` methods, for over-riding the auto-detected types for columns and specifying additional columns that should be added when the table is created. See :ref:`python_api_custom_columns`. (:issue:`100`) .. _v2_6: 2.6 (2020-04-15) ---------------- -- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (`#76 `__) +- New ``table.rows_where(..., order_by="age desc")`` argument, see :ref:`python_api_rows`. (:issue:`76`) .. _v2_5: 2.5 (2020-04-12) ---------------- -- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (`#96 `__) -- ``table.last_pk`` is now only available for inserts or upserts of a single record. (`#98 `__) -- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (`#97 `__) +- Panda's Timestamp is now stored as a SQLite TEXT column. Thanks, b0b5h4rp13! (:issue:`96`) +- ``table.last_pk`` is now only available for inserts or upserts of a single record. (:issue:`98`) +- New ``Database(filepath, recreate=True)`` parameter for deleting and recreating the database. (:issue:`97`) .. _v2_4_4: 2.4.4 (2020-03-23) ------------------ -- Fixed bug where columns with only null values were not correctly created. (`#95 `__) +- Fixed bug where columns with only null values were not correctly created. (:issue:`95`) .. _v2_4_3: 2.4.3 (2020-03-23) ------------------ -- Column type suggestion code is no longer confused by null values. (`#94 `__) +- Column type suggestion code is no longer confused by null values. (:issue:`94`) .. _v2_4_2: 2.4.2 (2020-03-14) ------------------ -- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (`#92 `__) +- ``table.column_dicts`` now works with all column types - previously it would throw errors on types other than ``TEXT``, ``BLOB``, ``INTEGER`` or ``FLOAT``. (:issue:`92`) - Documentation for ``NotFoundError`` thrown by ``table.get(pk)`` - see :ref:`python_api_get`. .. _v2_4_1: @@ -388,45 +566,45 @@ The theme of this release is better tools for working with binary data. The new 2.4.1 (2020-03-01) ------------------ -- ``table.enable_fts()`` now works with columns that contain spaces. (`#90 `__) +- ``table.enable_fts()`` now works with columns that contain spaces. (:issue:`90`) .. _v2_4: 2.4 (2020-02-26) ---------------- -- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (`#88 `__) -- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (`#88 `__) -- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (`#86 `__) -- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (`#87 `__) +- ``table.disable_fts()`` can now be used to remove FTS tables and triggers that were created using ``table.enable_fts(...)``. (:issue:`88`) +- The ``sqlite-utils disable-fts`` command can be used to remove FTS tables and triggers from the command-line. (:issue:`88`) +- Trying to create table columns with square braces ([ or ]) in the name now raises an error. (:issue:`86`) +- Subclasses of ``dict``, ``list`` and ``tuple`` are now detected as needing a JSON column. (:issue:`87`) .. _v2_3_1: 2.3.1 (2020-02-10) ------------------ -``table.create_index()`` now works for columns that contain spaces. (`#85 `__) +``table.create_index()`` now works for columns that contain spaces. (:issue:`85`) .. _v2_3: 2.3 (2020-02-08) ---------------- -``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (`#83 `__) +``table.exists()`` is now a method, not a property. This was not a documented part of the API before so I'm considering this a non-breaking change. (:issue:`83`) .. _v2_2_1: 2.2.1 (2020-02-06) ------------------ -Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (`#84 `__). +Fixed a bug where ``.upsert(..., hash_id="pk")`` threw an error (:issue:`84`). .. _v2_2: 2.2 (2020-02-01) ---------------- -New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (`#81 `__). +New feature: ``sqlite_utils.suggest_column_types([records])`` returns the suggested column types for a list of records. See :ref:`python_api_suggest_column_types`. (:issue:`81`). This replaces the undocumented ``table.detect_column_types()`` method. @@ -442,7 +620,7 @@ New feature: ``conversions={...}`` can be passed to the ``.insert()`` family of 2.0.1 (2020-01-05) ------------------ -The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (`#73 `__). +The ``.upsert()`` and ``.upsert_all()`` methods now raise a ``sqlite_utils.db.PrimaryKeyRequired`` exception if you call them without specifying the primary key column using ``pk=`` (:issue:`73`). .. _v2: @@ -464,14 +642,14 @@ For full background on this change, see `issue #66 `__) +- Fixed error thrown when ``.insert_all()`` and ``.upsert_all()`` were called with empty lists (:issue:`52`) .. _v1_12: 1.12 (2019-11-04) ----------------- -Python library utilities for deleting records (`#62 `__) +Python library utilities for deleting records (:issue:`62`) - ``db["tablename"].delete(4)`` to delete by primary key, see :ref:`python_api_delete` - ``db["tablename"].delete_where("id > ?", [3])`` to delete by a where clause, see :ref:`python_api_delete_where` @@ -485,14 +663,14 @@ Option to create triggers to automatically keep FTS tables up-to-date with newly - ``sqlite-utils enable-fts ... --create-triggers`` - see :ref:`Configuring full-text search using the CLI ` - ``db["tablename"].enable_fts(..., create_triggers=True)`` - see :ref:`Configuring full-text search using the Python library ` -- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (`#59 `__) +- Support for introspecting triggers for a database or table - see :ref:`python_api_introspection` (:issue:`59`) .. _v1_10: 1.10 (2019-08-23) ----------------- -Ability to introspect and run queries against views (`#54 `__) +Ability to introspect and run queries against views (:issue:`54`) - ``db.view_names()`` method and and ``db.views`` property - Separate ``View`` and ``Table`` classes, both subclassing new ``Queryable`` class @@ -505,21 +683,21 @@ See :ref:`python_api_views`. 1.9 (2019-08-04) ---------------- -- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (`#23 `__) +- ``table.m2m(...)`` method for creating many-to-many relationships: :ref:`python_api_m2m` (:issue:`23`) .. _v1_8: 1.8 (2019-07-28) ---------------- -- ``table.update(pk, values)`` method: :ref:`python_api_update` (`#35 `__) +- ``table.update(pk, values)`` method: :ref:`python_api_update` (:issue:`35`) .. _v1_7_1: 1.7.1 (2019-07-28) ------------------ -- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (`#50 `__) +- Fixed bug where inserting records with 11 columns in a batch of 100 triggered a "too many SQL variables" error (:issue:`50`) - Documentation and tests for ``table.drop()`` method: :ref:`python_api_drop` .. _v1_7: @@ -529,8 +707,8 @@ See :ref:`python_api_views`. Support for lookup tables. -- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (`#44 `__) -- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (`#46 `__) +- New ``table.lookup({...})`` utility method for building and querying lookup tables - see :ref:`python_api_lookup_tables` (:issue:`44`) +- New ``extracts=`` table configuration option, see :ref:`python_api_extracts` (:issue:`46`) - Use `pysqlite3 `__ if it is available, otherwise use ``sqlite3`` from the standard library - Table options can now be passed to the new ``db.table(name, **options)`` factory function in addition to being passed to ``insert_all(records, **options)`` and friends - see :ref:`python_api_table_configuration` - In-memory databases can now be created using ``db = Database(memory=True)`` @@ -540,19 +718,19 @@ Support for lookup tables. 1.6 (2019-07-18) ---------------- -- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (`#41 `__) +- ``sqlite-utils insert`` can now accept TSV data via the new ``--tsv`` option (:issue:`41`) .. _v1_5: 1.5 (2019-07-14) ---------------- -- Support for compound primary keys (`#36 `__) +- Support for compound primary keys (:issue:`36`) - Configure these using the CLI tool by passing ``--pk`` multiple times - In Python, pass a tuple of columns to the ``pk=(..., ...)`` argument: :ref:`python_api_compound_primary_keys` -- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (`#39 `__) +- New ``table.get()`` method for retrieving a record by its primary key: :ref:`python_api_get` (:issue:`39`) .. _v1_4_1: @@ -566,14 +744,14 @@ Support for lookup tables. 1.4 (2019-06-30) ---------------- -- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (`#33 `__) +- Added ``sqlite-utils index-foreign-keys`` command (:ref:`docs `) and ``db.index_foreign_keys()`` method (:ref:`docs `) (:issue:`33`) .. _v1_3: 1.3 (2019-06-28) ---------------- -- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (`#31 `__) +- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (:issue:`31`) .. _v1_2_2: @@ -587,15 +765,15 @@ Support for lookup tables. 1.2.1 (2019-06-20) ------------------ -- Check the column exists before attempting to add a foreign key (`#29 `__) +- Check the column exists before attempting to add a foreign key (:issue:`29`) .. _v1_2: 1.2 (2019-06-12) ---------------- -- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by instrospecting the database. See :ref:`python_api_add_foreign_key` for details. (`#25 `__) -- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (`#24 `__). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` +- Improved foreign key definitions: you no longer need to specify the ``column``, ``other_table`` AND ``other_column`` to define a foreign key - if you omit the ``other_table`` or ``other_column`` the script will attempt to guess the correct values by introspecting the database. See :ref:`python_api_add_foreign_key` for details. (:issue:`25`) +- Ability to set ``NOT NULL`` constraints and ``DEFAULT`` values when creating tables (:issue:`24`). Documentation: :ref:`Setting defaults and not null constraints (Python API) `, :ref:`Setting defaults and not null constraints (CLI) ` - Support for ``not_null_default=X`` / ``--not-null-default`` for setting a ``NOT NULL DEFAULT 'x'`` when adding a new column. Documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_1: @@ -603,8 +781,8 @@ Support for lookup tables. 1.1 (2019-05-28) ---------------- -- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key alread exists (`#21 `__) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` -- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (`#16 `__) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` +- Support for ``ignore=True`` / ``--ignore`` for ignoring inserted records if the primary key already exists (:issue:`21`) - documentation: :ref:`Inserting data (Python API) `, :ref:`Inserting data (CLI) ` +- Ability to add a column that is a foreign key reference using ``fk=...`` / ``--fk`` (:issue:`16`) - documentation: :ref:`Adding columns (Python API) `, :ref:`Adding columns (CLI) ` .. _v1_0_1: diff --git a/docs/cli.rst b/docs/cli.rst index 8a049a6..0c08526 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -8,25 +8,31 @@ The ``sqlite-utils`` command-line tool can be used to manipulate SQLite database .. contents:: :local: -.. _cli_query_json: +.. _cli_query: -Running queries and returning JSON -================================== +Running SQL queries +=================== -You can execute a SQL query against a database and get the results back as JSON like this:: +The ``sqlite-utils query`` command lets you run queries directly against a SQLite database file. This is the default subcommand, so the following two examples work the same way:: $ sqlite-utils query dogs.db "select * from dogs" + $ sqlite-utils dogs.db "select * from dogs" + +.. _cli_query_json: + +Returning JSON +-------------- + +The default format returned for queries is JSON:: + + $ sqlite-utils dogs.db "select * from dogs" [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] -This is the default command for ``sqlite-utils``, so you can instead use this:: +.. _cli_query_nl: - $ sqlite-utils dogs.db "select * from dogs" - -You can pass named parameters to the query using ``-p``:: - - $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 - [{":num * :num2": 30}] +Newline-delimited JSON +~~~~~~~~~~~~~~~~~~~~~~ Use ``--nl`` to get back newline-delimited JSON objects:: @@ -34,7 +40,12 @@ Use ``--nl`` to get back newline-delimited JSON objects:: {"id": 1, "age": 4, "name": "Cleo"} {"id": 2, "age": 2, "name": "Pancakes"} -You can use ``--arrays`` to request ararys instead of objects:: +.. _cli_query_arrays: + +JSON arrays +~~~~~~~~~~~ + +You can use ``--arrays`` to request arrays instead of objects:: $ sqlite-utils dogs.db "select * from dogs" --arrays [[1, 4, "Cleo"], @@ -62,6 +73,11 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_binary_json: + +Binary data in JSON +~~~~~~~~~~~~~~~~~~~ + Binary strings are not valid JSON, so BLOB columns containing binary data will be returned as a JSON object containing base64 encoded data, that looks like this:: $ sqlite-utils dogs.db "select name, content from images" | python -mjson.tool @@ -75,25 +91,11 @@ Binary strings are not valid JSON, so BLOB columns containing binary data will b } ] -If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the comand will return the number of affected rows:: - - $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" - [{"rows_affected": 1}] - -You can run queries against a temporary in-memory database by passing ``:memory:`` as the filename:: - - $ sqlite-utils :memory: "select sqlite_version()" - [{"sqlite_version()": "3.29.0"}] - -You can load SQLite extension modules using the `--load-extension` option:: - - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=/usr/local/lib/mod_spatialite.dylib - [{"spatialite_version()": "4.3.0a"}] .. _cli_json_values: Nested JSON values ------------------- +~~~~~~~~~~~~~~~~~~ If one of your columns contains JSON, by default it will be returned as an escaped string:: @@ -124,24 +126,10 @@ You can use the ``--json-cols`` option to automatically detect these JSON column } ] -.. _cli_attach: - -Attaching additional databases ------------------------------- - -SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. - -You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. - -This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: - - sqlite-utils dogs.db --attach books books.db \ - 'select * from sqlite_master union all select * from books.sqlite_master' - .. _cli_query_csv: -Running queries and returning CSV -================================= +Returning CSV or TSV +-------------------- You can use the ``--csv`` option to return results as CSV:: @@ -165,8 +153,8 @@ Use ``--tsv`` instead of ``--csv`` to get back tab-separated values:: .. _cli_query_table: -Running queries and outputting a table -====================================== +Table-formatted output +---------------------- You can use the ``--table`` option (or ``-t`` shortcut) to output query results as a table:: @@ -190,8 +178,8 @@ For a full list of table format options, run ``sqlite-utils query --help``. .. _cli_query_raw: -Returning raw data from a query, such as binary content -======================================================= +Returning raw data, such as binary content +------------------------------------------ If your table contains binary data in a ``BLOB`` you can use the ``--raw`` option to output specific columns directly to standard out. @@ -199,6 +187,182 @@ For example, to retrieve a binary image from a ``BLOB`` column and store it in a $ sqlite-utils photos.db "select contents from photos where id=1" --raw > myphoto.jpg + +.. _cli_query_parameters: + +Using named parameters +---------------------- + +You can pass named parameters to the query using ``-p``:: + + $ sqlite-utils query dogs.db "select :num * :num2" -p num 5 -p num2 6 + [{":num * :num2": 30}] + +These will be correctly quoted and escaped in the SQL query, providing a safe way to combine other values with SQL. + +.. _cli_query_update_insert_delete: + +UPDATE, INSERT and DELETE +------------------------- + +If you execute an ``UPDATE``, ``INSERT`` or ``DELETE`` query the command will return the number of affected rows:: + + $ sqlite-utils dogs.db "update dogs set age = 5 where name = 'Cleo'" + [{"rows_affected": 1}] + +SQLite extensions +----------------- + +You can load SQLite extension modules using the ``--load-extension`` option, see :ref:`cli_load_extension`. + +:: + + $ sqlite-utils dogs.db "select spatialite_version()" --load-extension=spatialite + [{"spatialite_version()": "4.3.0a"}] + +.. _cli_query_attach: + +Attaching additional databases +------------------------------ + +SQLite supports cross-database SQL queries, which can join data from tables in more than one database file. + +You can attach one or more additional databases using the ``--attach`` option, providing an alias to use for that database and the path to the SQLite file on disk. + +This example attaches the ``books.db`` database under the alias ``books`` and then runs a query that combines data from that database with the default ``dogs.db`` database:: + + sqlite-utils dogs.db --attach books books.db \ + 'select * from sqlite_master union all select * from books.sqlite_master' + +.. _cli_memory: + +Querying data directly using an in-memory database +================================================== + +The ``sqlite-utils memory`` command works similar to ``sqlite-utils query``, but allows you to execute queries against an in-memory database. + +You can also pass this command CSV or JSON files which will be loaded into a temporary in-memory table, allowing you to execute SQL against that data without a separate step to first convert it to SQLite. + +Without any extra arguments, this command executes SQL against the in-memory database directly:: + + $ sqlite-utils memory 'select sqlite_version()' + [{"sqlite_version()": "3.35.5"}] + +It takes all of the same output formatting options as :ref:`sqlite-utils query `: ``--csv`` and ``--csv`` and ``--table`` and ``--nl``:: + + $ sqlite-utils memory 'select sqlite_version()' --csv + sqlite_version() + 3.35.5 + $ sqlite-utils memory 'select sqlite_version()' --table --fmt grid + +--------------------+ + | sqlite_version() | + +====================+ + | 3.35.5 | + +--------------------+ + +.. _cli_memory_csv_json: + +Running queries directly against CSV or JSON +-------------------------------------------- + +If you have data in CSV or JSON format you can load it into an in-memory SQLite database and run queries against it directly in a single command using ``sqlite-utils memory`` like this:: + + $ sqlite-utils memory data.csv "select * from data" + +You can pass multiple files to the command if you want to run joins between data from different files:: + + $ sqlite-utils memory one.csv two.json "select * from one join two on one.id = two.other_id" + +If your data is JSON it should be the same format supported by the :ref:`sqlite-utils insert command ` - so either a single JSON object (treated as a single row) or a list of JSON objects. + +CSV data can be comma- or tab- delimited. + +The in-memory tables will be named after the files without their extensions. The tool also sets up aliases for those tables (using SQL views) as ``t1``, ``t2`` and so on, or you can use the alias ``t`` to refer to the first table:: + + $ sqlite-utils memory example.csv "select * from t" + +To read from standard input, use either ``-`` or ``stdin`` as the filename - then use ``stdin`` or ``t`` or ``t1`` as the table name:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" + +Incoming CSV data will be assumed to use ``utf-8``. If your data uses a different character encoding you can specify that with ``--encoding``:: + + $ cat example.csv | sqlite-utils memory - "select * from stdin" --encoding=latin-1 + +If you are joining across multiple CSV files they must all use the same encoding. + +Column types will be automatically detected in CSV or TSV data, using the same mechanism as ``--detect-types`` described in :ref:`cli_insert_csv_tsv`. You can pass the ``--no-detect-types`` option to disable this automatic type detection and treat all CSV and TSV columns as ``TEXT``. + +.. _cli_memory_explicit: + +Explicitly specifying the format +-------------------------------- + +By default, ``sqlite-utils memory`` will attempt to detect the incoming data format (JSON, TSV or CSV) automatically. + +You can instead specify an explicit format by adding a ``:csv``, ``:tsv``, ``:json`` or ``:nl`` (for newline-delimited JSON) suffix to the filename. For example:: + + $ sqlite-utils memory one.dat:csv two.dat:nl "select * from one union select * from two" + +Here the contents of ``one.dat`` will be treated as CSV and the contents of ``two.dat`` will be treated as newline-delimited JSON. + +To explicitly specify the format for data piped into the tool on standard input, use ``stdin:format`` - for example:: + + $ cat one.dat | sqlite-utils memory stdin:csv "select * from stdin" + +.. _cli_memory_attach: + +Joining in-memory data against existing databases using \-\-attach +------------------------------------------------------------------ + +The :ref:`attach option ` can be used to attach database files to the in-memory connection, enabling joins between in-memory data loaded from a file and tables in existing SQLite database files. An example:: + + $ echo "id\n1\n3\n5" | sqlite-utils memory - --attach trees trees.db \ + "select * from trees.trees where rowid in (select id from stdin)" + +Here the ``--attach trees trees.db`` option makes the ``trees.db`` database available with an alias of ``trees``. + +``select * from trees.trees where ...`` can then query the ``trees`` table in that database. + +The CSV data that was piped into the script is available in the ``stdin`` table, so ``... where rowid in (select id from stdin)`` can be used to return rows from the ``trees`` table that match IDs that were piped in as CSV content. + +.. _cli_memory_schema_dump_save: + +\-\-schema, \-\-dump and \-\-save +--------------------------------- + +To see the schema that will be created for a file or multiple files, use ``--schema``:: + + % sqlite-utils memory dogs.csv --schema + CREATE TABLE [dogs] ( + [id] INTEGER, + [age] INTEGER, + [name] TEXT + ); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + +You can output SQL that will both create the tables and insert the full data used to populate the in-memory database using ``--dump``:: + + % sqlite-utils memory dogs.csv --dump + BEGIN TRANSACTION; + CREATE TABLE [dogs] ( + [id] INTEGER, + [age] INTEGER, + [name] TEXT + ); + INSERT INTO "dogs" VALUES('1','4','Cleo'); + INSERT INTO "dogs" VALUES('2','2','Pancakes'); + CREATE VIEW t1 AS select * from [dogs]; + CREATE VIEW t AS select * from [dogs]; + COMMIT; + +Passing ``--save other.db`` will instead use that SQL to populate a new database file:: + + % sqlite-utils memory dogs.csv --save dogs.db + +These features are mainly intended as debugging tools - for much more finely grained control over how data is inserted into a SQLite database file see :ref:`cli_inserting_data` and :ref:`cli_insert_csv_tsv`. + .. _cli_rows: Returning all rows in a table @@ -268,7 +432,7 @@ Use ``--schema`` to include the schema of each table:: [age] INTEGER, [name] TEXT) -The ``--nl``, ``--csv``, ``--tsv`` and ``--table`` options are all available. +The ``--nl``, ``--csv``, ``--tsv``, ``--table`` and ``--fmt`` options are also available. .. _cli_views: @@ -293,6 +457,33 @@ It takes the same options as the ``tables`` command: * ``--tsv`` * ``--table`` +.. _cli_indexes: + +Listing indexes +=============== + +The ``indexes`` command lists any indexes configured for the database:: + + $ sqlite-utils indexes covid.db --table + table index_name seqno cid name desc coll key + -------------------------------- ------------------------------------------------------ ------- ----- ----------------- ------ ------ ----- + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_combined_key 0 12 combined_key 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_country_or_region 0 1 country_or_region 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_province_or_state 0 2 province_or_state 0 BINARY 1 + johns_hopkins_csse_daily_reports idx_johns_hopkins_csse_daily_reports_day 0 0 day 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_date 0 0 date 1 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_fips 0 3 fips 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_county 0 1 county 0 BINARY 1 + ny_times_us_counties idx_ny_times_us_counties_state 0 2 state 0 BINARY 1 + +It shows indexes across all tables. To see indexes for specific tables, list those after the database:: + + $ sqlite-utils indexes covid.db johns_hopkins_csse_daily_reports --table + +The command defaults to only showing the columns that are explicitly part of the index. To also include auxiliary columns use the ``--aux`` option - these columns will be listed with a ``key`` of ``0``. + +The command takes the same format options as the ``tables`` and ``views`` commands. + .. _cli_triggers: Listing triggers @@ -321,6 +512,24 @@ It defaults to showing triggers for all tables. To see triggers for one or more The command takes the same format options as the ``tables`` and ``views`` commands. +.. _cli_schema: + +Showing the schema +================== + +The ``sqlite-utils schema`` command shows the full SQL schema for the database:: + + $ sqlite-utils schema dogs.db + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + +This will show the schema for every table and index in the database. To view the schema just for a specified subset of tables pass those as additional arguments:: + + $ sqlite-utils schema dogs.db dogs chickens + ... + .. _cli_analyze_tables: Analyzing tables @@ -408,6 +617,21 @@ The ``_analyze_tables_`` table has the following schema:: PRIMARY KEY ([table], [column]) ); +The ``most_common`` and ``least_common`` columns will contain nested JSON arrays of the most common and least common values that look like this:: + + [ + ["Del Libertador, Av", 5068], + ["Alberdi Juan Bautista Av.", 4612], + ["Directorio Av.", 4552], + ["Rivadavia, Av", 4532], + ["Yerbal", 4512], + ["Cosquín", 4472], + ["Estado Plurinacional de Bolivia", 4440], + ["Gordillo Timoteo", 4424], + ["Montiel", 4360], + ["Condarco", 4288] + ] + .. _cli_inserting_data: Inserting JSON data @@ -445,6 +669,23 @@ If you feed it a JSON list it will insert multiple records. For example, if ``do } ] +You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: + + $ sqlite-utils insert dogs.db dogs dogs.json --pk=id + +You can skip inserting any records that have a primary key that already exists using ``--ignore``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --ignore + +You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: + + $ sqlite-utils insert dogs.db dogs dogs.json --truncate + +.. _cli_inserting_data_binary: + +Inserting binary data +--------------------- + You can insert binary data into a BLOB column by first encoding it using base64 and then structuring it like this:: [ @@ -457,17 +698,10 @@ You can insert binary data into a BLOB column by first encoding it using base64 } ] -You can import all three records into an automatically created ``dogs`` table and set the ``id`` column as the primary key like so:: +.. _cli_inserting_data_nl_json: - $ sqlite-utils insert dogs.db dogs dogs.json --pk=id - -You can skip inserting any records that have a primary key that already exists using ``--ignore``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --ignore - -You can delete all the existing rows in the table before inserting the new records using ``--truncate``:: - - $ sqlite-utils insert dogs.db dogs dogs.json --truncate +Inserting newline-delimited JSON +-------------------------------- You can also import newline-delimited JSON using the ``--nl`` option. Since `Datasette `__ can export newline-delimited JSON, you can combine the two tools like so:: @@ -488,6 +722,49 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite 207368,920 Kirkham St,37.760210314285,-122.47073935813 188702,1501 Evans Ave,37.7422086702947,-122.387293152263 +.. _cli_inserting_data_flatten: + +Flattening nested JSON objects +------------------------------ + +``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. + +If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. + +Consider this example document, in a file called ``log.json``:: + + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + "requestSize": "534", + "status": 200 + }, + "insertId": "6111722f000b5b4c4d4071e2", + "labels": { + "service": "datasette-io" + } + } + +Inserting this into a table using ``sqlite-utils insert logs.db logs log.json`` will create a table with the following schema:: + + CREATE TABLE [logs] ( + [httpRequest] TEXT, + [insertId] TEXT, + [labels] TEXT + ); + +With the ``--flatten`` option columns will be created using ``topkey_nextkey`` column names - so running ``sqlite-utils insert logs.db logs log.json --flatten`` will create the following schema instead:: + + CREATE TABLE [logs] ( + [httpRequest_latency] TEXT, + [httpRequest_requestMethod] TEXT, + [httpRequest_requestSize] TEXT, + [httpRequest_status] INTEGER, + [insertId] TEXT, + [labels_service] TEXT + ); + .. _cli_insert_csv_tsv: Inserting CSV or TSV data @@ -507,6 +784,31 @@ Data is expected to be encoded as Unicode UTF-8. If your data is an another char A progress bar is displayed when inserting data from a file. You can hide the progress bar using the ``--silent`` option. +By default every column inserted from a CSV or TSV file will be of type ``TEXT``. To automatically detect column types - resulting in a mix of ``TEXT``, ``INTEGER`` and ``FLOAT`` columns, use the ``--detect-types`` option (or its shortcut ``-d``). + +For example, given a ``creatures.csv`` file containing this:: + + name,age,weight + Cleo,6,45.5 + Dori,1,3.5 + +The following command:: + + $ sqlite-utils insert creatures.db creatures creatures.csv --csv --detect-types + +Will produce this schema:: + + $ sqlite-utils schema creatures.db + CREATE TABLE "creatures" ( + [name] TEXT, + [age] INTEGER, + [weight] FLOAT + ); + +You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ``--detect-types`` to be the default behavior:: + + $ export SQLITE_UTILS_DETECT_TYPES=1 + .. _cli_insert_csv_tsv_delimiter: Alternative delimiters and quote characters @@ -661,6 +963,128 @@ The ``-`` argument indicates data should be read from standard input. The string When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. +.. _cli_convert: + +Converting data in columns +========================== + +The ``convert`` command can be used to transform the data in a specified column - for example to parse a date string into an ISO timestamp, or to split a string of tags into a JSON array. + +The command accepts a database, table, one or more columns and a string of Python code to be executed against the values from those columns. The following example would replace the values in the ``headline`` column in the ``articles`` table with an upper-case version:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' + +The Python code is passed as a string. Within that Python code the ``value`` variable will be the value of the current column. + +The code you provide will be compiled into a function that takes ``value`` as a single argument. If you break your function body into multiple lines the last line should be a ``return`` statement:: + + $ sqlite-utils convert content.db articles headline ' + value = str(value) + return value.upper()' + +You can specify Python modules that should be imported and made available to your code using one or more ``--import`` options. This example uses the ``textwrap`` module to wrap the ``content`` column at 100 characters:: + + $ sqlite-utils convert content.db articles content \ + '"\n".join(textwrap.wrap(value, 100))' \ + --import=textwrap + +The transformation will be applied to every row in the specified table. You can limit that to just rows that match a ``WHERE`` clause using ``--where``:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like '%cat%'" + +You can include named parameters in your where clause and populate them using one or more ``--param`` options:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --where "headline like :like" \ + --param like '%cat%' + +The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. + +.. _cli_convert_recipes: + +sqlite-utils convert recipes +---------------------------- + +Various built-in recipe functions are available for common operations. These are: + +``r.jsonsplit(value, delimiter=',', type=)`` + Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]`` + + The ``delimiter`` parameter can be used to specify a different delimiter. + + The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4.5`` the following:: + + r.jsonsplit(value, type=float) + + Would produce an array like this: ``[1.2, 3.0, 4.5]`` + +``r.parsedate(value, dayfirst=False, yearfirst=False)`` + Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` + + In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` + +These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: + + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value)' + +To use any of the documented parameters, do this:: + + $ sqlite-utils convert my.db mytable mycolumn \ + 'r.jsonsplit(value, delimiter=":")' + +.. _cli_convert_output: + +Saving the result to a different column +--------------------------------------- + +The ``--output`` and ``--output-type`` options can be used to save the result of the conversion to a separate column, which will be created if that column does not already exist:: + + $ sqlite-utils convert content.db articles headline 'value.upper()' \ + --output headline_upper + +The type of the created column defaults to ``text``, but a different column type can be specified using ``--output-type``. This example will create a new floating point column called ``id_as_a_float`` with a copy of each item's ID increased by 0.5:: + + $ sqlite-utils convert content.db articles id 'float(value) + 0.5' \ + --output id_as_a_float \ + --output-type float + +You can drop the original column at the end of the operation by adding ``--drop``. + +.. _cli_convert_multi: + +Converting a column into multiple columns +----------------------------------------- + +Sometimes you may wish to convert a single column into multiple derived columns. For example, you may have a ``location`` column containing ``latitude,longitude`` values which you wish to split out into separate ``latitude`` and ``longitude`` columns. + +You can achieve this using the ``--multi`` option to ``sqlite-utils convert``. This option expects your Python code to return a Python dictionary: new columns well be created and populated for each of the keys in that dictionary. + +For the ``latitude,longitude`` example you would use the following:: + + $ sqlite-utils convert demo.db places location \ + 'bits = value.split(",") + return { + "latitude": float(bits[0]), + "longitude": float(bits[1]), + }' --multi + +The type of the returned values will be taken into account when creating the new columns. In this example, the resulting database schema will look like this: + +.. code-block:: sql + + CREATE TABLE [places] ( + [location] TEXT, + [latitude] FLOAT, + [longitude] FLOAT + ); + +The code function can also return ``None``, in which case its output will be ignored. You can drop the original column at the end of the operation by adding ``--drop``. + .. _cli_create_table: Creating tables @@ -861,45 +1285,44 @@ Here's a more complex example that makes use of these options. It converts `this --fk-column country_id \ --rename country_long name -After running the above, the command ``sqlite3 global.db .schema`` reveals the following schema: +After running the above, the command ``sqlite-utils schema global.db`` reveals the following schema: .. code-block:: sql CREATE TABLE [countries] ( - [id] INTEGER PRIMARY KEY, - [country] TEXT, - [name] TEXT + [id] INTEGER PRIMARY KEY, + [country] TEXT, + [name] TEXT + ); + CREATE TABLE "power_plants" ( + [country_id] INTEGER, + [name] TEXT, + [gppd_idnr] TEXT, + [capacity_mw] TEXT, + [latitude] TEXT, + [longitude] TEXT, + [primary_fuel] TEXT, + [other_fuel1] TEXT, + [other_fuel2] TEXT, + [other_fuel3] TEXT, + [commissioning_year] TEXT, + [owner] TEXT, + [source] TEXT, + [url] TEXT, + [geolocation_source] TEXT, + [wepp_id] TEXT, + [year_of_capacity_data] TEXT, + [generation_gwh_2013] TEXT, + [generation_gwh_2014] TEXT, + [generation_gwh_2015] TEXT, + [generation_gwh_2016] TEXT, + [generation_gwh_2017] TEXT, + [generation_data_source] TEXT, + [estimated_generation_gwh] TEXT, + FOREIGN KEY([country_id]) REFERENCES [countries]([id]) ); CREATE UNIQUE INDEX [idx_countries_country_name] ON [countries] ([country], [name]); - CREATE TABLE IF NOT EXISTS "power_plants" ( - [rowid] INTEGER PRIMARY KEY, - [country_id] INTEGER, - [name] TEXT, - [gppd_idnr] TEXT, - [capacity_mw] TEXT, - [latitude] TEXT, - [longitude] TEXT, - [primary_fuel] TEXT, - [other_fuel1] TEXT, - [other_fuel2] TEXT, - [other_fuel3] TEXT, - [commissioning_year] TEXT, - [owner] TEXT, - [source] TEXT, - [url] TEXT, - [geolocation_source] TEXT, - [wepp_id] TEXT, - [year_of_capacity_data] TEXT, - [generation_gwh_2013] TEXT, - [generation_gwh_2014] TEXT, - [generation_gwh_2015] TEXT, - [generation_gwh_2016] TEXT, - [generation_gwh_2017] TEXT, - [generation_data_source] TEXT, - [estimated_generation_gwh] TEXT, - FOREIGN KEY(country_id) REFERENCES countries(id) - ); .. _cli_create_view: @@ -1033,6 +1456,14 @@ Use the ``--unique`` option to create a unique index. Use ``--if-not-exists`` to avoid attempting to create the index if one with that name already exists. +To add an index on a column in descending order, prefix the column with a hyphen. Since this can be confused for a command-line option you need to construct that like this:: + + $ sqlite-utils create-index mydb.db mytable -- col1 -col2 col3 + +This will create an index on that table on ``(col1, col2 desc, col3)``. + +If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. + .. _cli_fts: Configuring full-text search @@ -1181,16 +1612,29 @@ You can disable WAL mode using ``disable-wal``:: Both of these commands accept one or more database files as arguments. +.. _cli_dump: + +Dumping the database to SQL +=========================== + +The ``dump`` command outputs a SQL dump of the schema and full contents of the specified database file:: + + $ sqlite-utils dump mydb.db + BEGIN TRANSACTION; + CREATE TABLE ... + ... + COMMIT; + .. _cli_load_extension: Loading SQLite extensions ========================= -Many of these commands have the ablity to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. +Many of these commands have the ability to load additional SQLite extensions using the ``--load-extension=/path/to/extension`` option - use ``--help`` to check for support, e.g. ``sqlite-utils rows --help``. This option can be applied multiple times to load multiple extensions. Since `SpatiaLite `__ is commonly used with SQLite, the value ``spatialite`` is special: it will search for SpatiaLite in the most common installation locations, saving you from needing to remember exactly where that module is located:: - $ sqlite-utils :memory: "select spatialite_version()" --load-extension=spatialite + $ sqlite-utils memory "select spatialite_version()" --load-extension=spatialite [{"spatialite_version()": "4.3.0a"}] diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt new file mode 100644 index 0000000..f8418c4 --- /dev/null +++ b/docs/codespell-ignore-words.txt @@ -0,0 +1 @@ +doub diff --git a/docs/conf.py b/docs/conf.py index 929a41e..1f5a158 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,12 @@ from subprocess import Popen, PIPE # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +autodoc_member_order = "bysource" + +extlinks = { + "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#"), +} # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..ec1a4a7 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,67 @@ +.. _contributing: + +============== + Contributing +============== + +To work on this library locally, first checkout the code. Then create a new virtual environment:: + + git clone git@github.com:simonw/sqlite-utils + cd sqlite-utils + python3 -mvenv venv + source venv/bin/activate + +Or if you are using ``pipenv``:: + + pipenv shell + +Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment. + +.. _contributing_tests: + +Running the tests +================= + +To install the dependencies and test dependencies:: + + pip install -e '.[test]' + +To run the tests:: + + pytest + +.. _contributing_docs: + +Building the documentation +========================== + +To build the documentation, first install the documentation dependencies:: + + pip install -e '.[docs]' + +Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file:: + + cd docs + make livehtml + +.. _contributing_linting: + +Linting and formatting +====================== + +``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking. + +Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project:: + + black . + +To install ``mypy`` and ``flake8`` run the following:: + + pip install -e '.[flake8,mypy]' + +Both commands can then be run in the root of the project like this:: + + flake8 + mypy sqlite_utils + +All three of these tools are run by our CI mechanism against every commit and pull request. diff --git a/docs/dogs.db b/docs/dogs.db deleted file mode 100644 index e69de29..0000000 diff --git a/docs/index.rst b/docs/index.rst index 571020f..0629e0e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,7 +13,7 @@ .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE -*Python utility functions for manipulating SQLite databases* +*CLI tool and Python utility functions for manipulating SQLite databases* This library and command-line utility helps create SQLite databases from an existing collection of data. @@ -29,8 +29,9 @@ Contents .. toctree:: :maxdepth: 3 + installation cli python-api + reference + contributing changelog - -Take a look at `this script `_ for an example of this library in action. diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..f4f132e --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,40 @@ +.. _installation: + +============== + Installation +============== + +``sqlite-utils`` is tested on Linux, macOS and Windows. + +.. _installation_homebrew: + +Using Homebrew +============== + +The :ref:`sqlite-utils command-line tool ` can be installed on macOS using Homebrew:: + + brew install sqlite-utils + +If you have it installed and want to upgrade to the most recent release, you can run:: + + brew upgrade sqlite-utils + +Then run ``sqlite-utils --version`` to confirm the installed version. + +.. _installation_pip: + +Using pip +========= + +The `sqlite-utils package `__ on PyPI includes both the :ref:`sqlite_utils Python library ` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so:: + + pip install sqlite-utils + +.. _installation_pipx: + +Using pipx +========== + +`pipx `__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this:: + + pipx install sqlite-utils diff --git a/docs/python-api.rst b/docs/python-api.rst index e5ffd44..b793920 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1,8 +1,8 @@ .. _python_api: -============ - Python API -============ +============================= + sqlite_utils Python library +============================= .. contents:: :local: @@ -59,12 +59,11 @@ You can attach an additional database using the ``.attach()`` method, providing db = Database("first.db") db.attach("second", "second.db") # Now you can run queries like this one: - cursor = db.execute(""" + print(db.query(""" select * from table_in_first union all select * from second.table_in_second - """) - print(cursor.fetchall()) + """)) You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. @@ -97,27 +96,77 @@ You can also turn on a tracer function temporarily for a block of code using the This example will print queries only for the duration of the ``with`` block. -.. _python_api_execute: +.. _python_api_executing_queries: Executing queries ================= -The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the tracer function if one has been registered. +The ``Database`` class offers several methods for directly executing SQL queries. + +.. _python_api_query: + +db.query(sql, params) +--------------------- + +The ``db.query(sql)`` function executes a SQL query and returns an iterator over Python dictionaries representing the resulting rows: + +.. code-block:: python + + db = Database(memory=True) + db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + for row in db.query("select * from dogs"): + print(row) + # Outputs: + # {'name': 'Cleo'} + # {'name': 'Pancakes'} + +.. _python_api_execute: + +db.execute(sql, params) +----------------------- + +The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around ``.execute()`` and ``.executescript()`` on the underlying SQLite connection. These wrappers log to the :ref:`tracer function ` if one has been registered. + +``db.execute(sql)`` returns a `sqlite3.Cursor `__ that was used to execute the SQL. .. code-block:: python db = Database(memory=True) db["dogs"].insert({"name": "Cleo"}) - db.execute("update dogs set name = 'Cleopaws'") + cursor = db.execute("update dogs set name = 'Cleopaws'") + print(cursor.rowcount) + # Outputs the number of rows affected by the update + # In this case 2 -You can pass parameters as an optional second argument, using either a list or a dictionary. These will be correctly quoted and escaped. +Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. + +.. _python_api_parameters: + +Passing parameters +------------------ + +Both ``db.query()`` and ``db.execute()`` accept an optional second argument for parameters to be passed to the SQL query. + +This can take the form of either a tuple/list or a dictionary, depending on the type of parameters used in the query. Values passed in this way will be correctly quoted and escaped, helping avoid XSS vulnerabilities. + +``?`` parameters in the SQL query can be filled in using a list: .. code-block:: python - # Using ? and a list: db.execute("update dogs set name = ?", ["Cleopaws"]) - # Or using :name and a dictionary: - db.execute("update dogs set name = :name", {"name": "Cleopaws"}) + # This will rename ALL dogs to be called "Cleopaws" + +Named parameters using ``:name`` can be filled using a dictionary: + +.. code-block:: python + + dog = next(db.query( + "select rowid, name from dogs where name = :name", + {"name": "Cleopaws"} + )) + # dog is now {'rowid': 1, 'name': 'Cleopaws'} + +In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method. .. _python_api_table: @@ -202,7 +251,13 @@ You can filter rows by a WHERE clause using ``.rows_where(where, where_args)``:: ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} -To return custom columns (instead of using ``select *``) pass ``select=``:: +The first argument is a fragment of SQL. The second, optional argument is values to be passed to that fragment - you can use ``?`` placeholders and pass an array, or you can use ``:named`` parameters and pass a dictionary, like this:: + + >>> for row in db["dogs"].rows_where("age > :age", {"age": 3}): + ... print(row) + {'id': 1, 'age': 4, 'name': 'Cleo'} + +To return custom columns (instead of the default that uses ``select *``) pass ``select="column1, column2"``:: >>> db = sqlite_utils.Database("dogs.db") >>> for row in db["dogs"].rows_where(select='name, age'): @@ -231,6 +286,16 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. _python_api_rows_count_where: + +Counting rows +------------- + +To count the number of rows that would be returned by a where filter, use ``.count_where(where, where_args)``: + + >>> db["dogs"].count_where("age > ?", [1]): + 2 + .. _python_api_pks_and_rows_where: Listing rows with their primary keys @@ -291,6 +356,21 @@ If the record does not exist a ``NotFoundError`` will be raised: except NotFoundError: print("Dog not found") +.. _python_api_schema: + +Showing the schema +================== + +The ``db.schema`` property returns the full SQL schema for the database as a string:: + + >>> db = sqlite_utils.Database("dogs.db") + >>> print(db.schema) + >>> print(db.schema) + CREATE TABLE "dogs" ( + [id] INTEGER PRIMARY KEY, + [name] TEXT + ); + .. _python_api_creating_tables: Creating tables @@ -603,7 +683,7 @@ The first argument to ``update()`` is the primary key. This can be a single valu >>> db["compound_dogs"].update((5, 3), {"name": "Updated"}) -The second argument is a dictonary of columns that should be updated, along with their new values. +The second argument is a dictionary of columns that should be updated, along with their new values. You can cause any missing columns to be added automatically using ``alter=True``:: @@ -632,7 +712,7 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> db["dogs"].delete_where("age < ?", [3]): + >>> db["dogs"].delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. @@ -647,7 +727,7 @@ For example, given the dogs database you could upsert the record for Cleo like s .. code-block:: python - db["dogs"].upsert([{ + db["dogs"].upsert({ "id": 1, "name": "Cleo", "twitter": "cleopaws", @@ -666,6 +746,53 @@ An ``upsert_all()`` method is also available, which behaves like ``insert_all()` .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. +.. _python_api_convert: + +Converting data in columns +========================== + +The ``table.convert(...)`` method can be used to apply a conversion function to the values in a column, either to update that column or to populate new columns. It is the Python library equivalent of the :ref:`sqlite-utils convert ` command. + +This feature works by registering a custom SQLite function that applies a Python transformation, then running a SQL query equivalent to ``UPDATE table SET column = convert_value(column);`` + +To transform a specific column to uppercase, you would use the following: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper()) + +You can pass a list of columns, in which case the transformation will be applied to each one: + +.. code-block:: python + + db["dogs"].convert(["name", "twitter"], lambda value: value.upper()) + +To save the output to of the transformation to a different column, use the ``output=`` parameter: + +.. code-block:: python + + db["dogs"].convert("name", lambda value: value.upper(), output="name_upper") + +This will add the new column, if it does not already exist. You can pass ``output_type=int`` or some other type to control the type of the new column - otherwise it will default to text. + +If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``. + +You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column: + +.. code-block:: python + + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + +The ``.convert()`` method accepts optional ``where=`` and ``where_args=`` parameters which can be used to apply the conversion to a subset of rows specified by a where clause. Here's how to apply the conversion only to rows with an ``id`` that is higher than 20: + +.. code-block:: python + + table.convert("title", lambda v: v.upper(), where="id > :id", where_args={"id": 20}) + +These behave the same as the corresponding parameters to the :ref:`.rows_where() ` method, so you can use ``?`` placeholders and a list of values instead of ``:named`` placeholders with a dictionary. + .. _python_api_lookup_tables: Working with lookup tables @@ -835,7 +962,7 @@ The ``table.analyze_column(column, common_limit=10, value_truncate=None)`` metho The name of the column ``total_rows`` - The total number of rows in the table` + The total number of rows in the table ``num_null`` The number of rows for which this column is null @@ -969,7 +1096,7 @@ Here's an example of this mechanism in action: ]) db["books"].add_foreign_key("author_id", "authors", "id") -The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you ommit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: +The ``table.add_foreign_key(column, other_table, other_column)`` method takes the name of the column, the table that is being referenced and the key column within that other table. If you omit the ``other_column`` argument the primary key from that table will be used automatically. If you omit the ``other_table`` argument the table will be guessed based on some simple rules: - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` @@ -1050,6 +1177,9 @@ The ``table.transform()`` method can do all of these things, by implementing a m The ``.transform()`` method takes a number of parameters, all of which are optional. +Altering column types +--------------------- + To alter the type of a column, use the ``types=`` argument: .. code-block:: python @@ -1057,6 +1187,11 @@ To alter the type of a column, use the ``types=`` argument: # Convert the 'age' column to an integer, and 'weight' to a float table.transform(types={"age": int, "weight": float}) +See :ref:`python_api_add_column` for a list of available types. + +Renaming columns +---------------- + The ``rename=`` parameter can rename columns: .. code-block:: python @@ -1064,6 +1199,9 @@ The ``rename=`` parameter can rename columns: # Rename 'age' to 'initial_age': table.transform(rename={"age": "initial_age"}) +Dropping columns +---------------- + To drop columns, pass them in the ``drop=`` set: .. code-block:: python @@ -1071,6 +1209,9 @@ To drop columns, pass them in the ``drop=`` set: # Drop the 'age' column: table.transform(drop={"age"}) +Changing primary keys +--------------------- + To change the primary key for a table, use ``pk=``. This can be passed a single column for a regular primary key, or a tuple of columns to create a compound primary key. Passing ``pk=None`` will remove the primary key and convert the table into a ``rowid`` table. .. code-block:: python @@ -1078,6 +1219,9 @@ To change the primary key for a table, use ``pk=``. This can be passed a single # Make `user_id` the new primary key table.transform(pk="user_id") +Changing not null status +------------------------ + You can change the ``NOT NULL`` status of columns by using ``not_null=``. You can pass this a set of columns to make those columns ``NOT NULL``: .. code-block:: python @@ -1095,6 +1239,9 @@ If you want to take existing ``NOT NULL`` columns and change them to allow null # Make age allow NULL and switch weight to being NOT NULL: table.transform(not_null={"age": False, "weight": True}) +Altering column defaults +------------------------ + The ``defaults=`` parameter can be used to set or change the defaults for different columns: .. code-block:: python @@ -1105,6 +1252,9 @@ The ``defaults=`` parameter can be used to set or change the defaults for differ # Now remove the default from that column: table.transform(defaults={"age": None}) +Changing column order +--------------------- + The ``column_order=`` parameter can be used to change the order of the columns. If you pass the names of a subset of the columns those will go first and columns you omitted will appear in their existing order after them. .. code-block:: python @@ -1112,6 +1262,9 @@ The ``column_order=`` parameter can be used to change the order of the columns. # Change column order table.transform(column_order=("name", "age", "id") +Dropping foreign key constraints +-------------------------------- + You can use ``.transform()`` to remove foreign key constraints from a table. This example drops two foreign keys - the one from ``places.country`` to ``country.id`` and the one from ``places.continent`` to ``continent.id``: @@ -1443,6 +1596,11 @@ If you have loaded an existing table or view, you can use introspection to find >>> db["PlantType"] +.. _python_api_introspection_exists: + +.exists() +--------- + The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType"].exists() @@ -1450,6 +1608,11 @@ The ``.exists()`` method can be used to find out if a table exists or not:: >>> db["PlantType2"].exists() False +.. _python_api_introspection_count: + +.count +------ + The ``.count`` property shows the current number of rows (``select count(*) from table``):: >>> db["PlantType"].count @@ -1457,25 +1620,60 @@ The ``.count`` property shows the current number of rows (``select count(*) from >>> db["Street_Tree_List"].count 189144 -This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.execute_count()`` instead of accessing the property. +This property will take advantage of :ref:`python_api_cached_table_counts` if the ``use_counts_table`` property is set on the database. You can avoid that optimization entirely by calling ``table.count_where()`` instead of accessing the property. -The ``.columns`` property shows the columns in the table or view:: +.. _python_api_introspection_columns: + +.columns +-------- + +The ``.columns`` property shows the columns in the table or view. It returns a list of ``Column(cid, name, type, notnull, default_value, is_pk)`` named tuples. + +:: >>> db["PlantType"].columns [Column(cid=0, name='id', type='INTEGER', notnull=0, default_value=None, is_pk=1), Column(cid=1, name='value', type='TEXT', notnull=0, default_value=None, is_pk=0)] -The ``.columns_dict`` property returns a dictionary version of this with just the names and types:: +.. _python_api_introspection_columns_dict: + +.columns_dict +------------- + +The ``.columns_dict`` property returns a dictionary version of the columns with just the names and Python types:: >>> db["PlantType"].columns_dict {'id': , 'value': } +.. _python_api_introspection_pks: + +.pks +---- + The ``.pks`` property returns a list of strings naming the primary key columns for the table:: >>> db["PlantType"].pks ['id'] -The ``.foreign_keys`` property shows if the table has any foreign key relationships. It is not available on views. +If a table has no primary keys but is a `rowid table `__, this property will return ``['rowid']``. + +.. _python_api_introspection_use_rowid: + +.use_rowid +---------- + +Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly defined primary keys must use that ``rowid`` as the primary key for identifying individual rows. The ``.use_rowid`` property checks to see if a table needs to use the ``rowid`` in this way - it returns ``True`` if the table has no explicitly defined primary keys and ``False`` otherwise. + + >>> db["PlantType"].use_rowid + False + + +.. _python_api_introspection_foreign_keys: + +.foreign_keys +------------- + +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. :: @@ -1487,6 +1685,11 @@ The ``.foreign_keys`` property shows if the table has any foreign key relationsh ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] +.. _python_api_introspection_schema: + +.schema +------- + The ``.schema`` property outputs the table's schema as a SQL string:: >>> print(db["Street_Tree_List"].schema) @@ -1517,7 +1720,12 @@ The ``.schema`` property outputs the table's schema as a SQL string:: FOREIGN KEY ("qCareAssistant") REFERENCES [qCareAssistant](id), FOREIGN KEY ("qLegalStatus") REFERENCES [qLegalStatus](id)) -The ``.indexes`` property shows you all indexes created for a table. It is not available on views. +.. _python_api_introspection_indexes: + +.indexes +-------- + +The ``.indexes`` property returns all indexes created for a table, as a list of ``Index(seq, name, unique, origin, partial, columns)`` named tuples. It is not available on views. :: @@ -1529,7 +1737,39 @@ The ``.indexes`` property shows you all indexes created for a table. It is not a Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] -The ``.triggers`` property lists database triggers. It can be used on both database and table objects. +.. _python_api_introspection_xindexes: + +.xindexes +--------- + +The ``.xindexes`` property returns more detailed information about the indexes on the table, using the SQLite `PRAGMA index_xinfo() `__ mechanism. It returns a list of ``XIndex(name, columns)`` named tuples, where ``columns`` is a list of ``XIndexColumn(seqno, cid, name, desc, coll, key)`` named tuples. + +:: + + >>> db["ny_times_us_counties"].xindexes + [ + XIndex( + name='idx_ny_times_us_counties_date', + columns=[ + XIndexColumn(seqno=0, cid=0, name='date', desc=1, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ), + XIndex( + name='idx_ny_times_us_counties_fips', + columns=[ + XIndexColumn(seqno=0, cid=3, name='fips', desc=0, coll='BINARY', key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll='BINARY', key=0) + ] + ) + ] + +.. _python_api_introspection_triggers: + +.triggers +--------- + +The ``.triggers`` property lists database triggers. It can be used on both database and table objects. It returns a list of ``Trigger(name, table, sql)`` named tuples. :: @@ -1540,6 +1780,11 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db["authors"].triggers +.. _python_api_introspection_triggers_dict: + +.triggers_dict +-------------- + The ``.triggers_dict`` property returns the triggers for that table as a dictionary mapping their names to their SQL definitions. :: @@ -1558,6 +1803,11 @@ The same property exists on the database, and will return all triggers across al 'authors_ad': 'CREATE TRIGGER [authors_ad] AFTER DELETE...', 'authors_au': 'CREATE TRIGGER [authors_au] AFTER UPDATE'} +.. _python_api_introspection_detect_fts: + +.detect_fts() +------------- + The ``detect_fts()`` method returns the associated SQLite FTS table name, if one exists for this table. If the table has not been configured for full-text search it returns ``None``. :: @@ -1565,12 +1815,22 @@ The ``detect_fts()`` method returns the associated SQLite FTS table name, if one >>> db["authors"].detect_fts() "authors_fts" +.. _python_api_introspection_virtual_table_using: + +.virtual_table_using +-------------------- + The ``.virtual_table_using`` property reveals if a table is a virtual table. It returns ``None`` for regular tables and the upper case version of the type of virtual table otherwise. For example:: >>> db["authors"].enable_fts(["name"]) >>> db["authors_fts"].virtual_table_using "FTS5" +.. _python_api_introspection_has_counts_triggers: + +.has_counts_triggers +-------------------- + The ``.has_counts_triggers`` property shows if a table has been configured with triggers for updating a ``_counts`` table, as described in :ref:`python_api_cached_table_counts`. :: @@ -1848,6 +2108,8 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() +.. _python_api_create_index: + Creating indexes ================ @@ -1866,6 +2128,17 @@ By default the index will be named ``idx_{table-name}_{columns}`` - if you want index_name="good_dogs_by_age" ) +To create an index in descending order for a column, wrap the column name in ``db.DescIndex()`` like this: + +.. code-block:: python + + from sqlite_utils.db import DescIndex + + db["dogs"].create_index( + ["is_good_dog", DescIndex("age")], + index_name="good_dogs_by_age" + ) + You can create a unique index by passing ``unique=True``: .. code-block:: python @@ -2050,12 +2323,24 @@ If you want to deliberately replace the registered function with a new implement def reverse_string(s): return s[::-1] +Exceptions that occur inside a user-defined function default to returning the following error:: + + Unexpected error: user-defined function raised exception + +You can cause ``sqlite3`` to return more useful errors, including the traceback from the custom function, by executing the following before your custom functions are executed: + +.. code-block:: python + + from sqlite_utils.utils import sqlite3 + + sqlite3.enable_callback_tracebacks(True) + .. _python_api_quote: Quoting strings for use in SQL ============================== -In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.execute()``, as described in :ref:`python_api_execute`. +In almost all cases you should pass values to your SQL queries using the optional ``parameters`` argument to ``db.query()``, as described in :ref:`python_api_parameters`. If that option isn't relevant to your use-case you can to quote a string for use with SQLite using the ``db.quote()`` method, like so: diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 0000000..ea1203e --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,70 @@ +.. _reference: + +=============== + API Reference +=============== + +.. contents:: :local: + +.. _reference_db_database: + +sqlite_utils.db.Database +======================== + +.. autoclass:: sqlite_utils.db.Database + :members: + :undoc-members: + :special-members: __getitem__ + :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys + +.. _reference_db_queryable: + +sqlite_utils.db.Queryable +========================= + +:ref:`Table ` and :ref:`View ` are both subclasses of ``Queryable``, providing access to the following methods: + +.. autoclass:: sqlite_utils.db.Queryable + :members: + :undoc-members: + :exclude-members: execute_count + +.. _reference_db_table: + +sqlite_utils.db.Table +===================== + +.. autoclass:: sqlite_utils.db.Table + :members: + :undoc-members: + :show-inheritance: + :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns + +.. _reference_db_view: + +sqlite_utils.db.View +==================== + +.. autoclass:: sqlite_utils.db.View + :members: + :undoc-members: + :show-inheritance: + +.. _reference_db_other: + +Other +===== + +.. _reference_db_other_column: + +sqlite_utils.db.Column +---------------------- + +.. autoclass:: sqlite_utils.db.Column + +.. _reference_db_other_column_details: + +sqlite_utils.db.ColumnDetails +----------------------------- + +.. autoclass:: sqlite_utils.db.ColumnDetails diff --git a/docs/tutorial.ipynb b/docs/tutorial.ipynb new file mode 100644 index 0000000..aa22461 --- /dev/null +++ b/docs/tutorial.ipynb @@ -0,0 +1,1053 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27ae18ec", + "metadata": {}, + "source": [ + "# The sqlite-utils tutorial\n", + "\n", + "[sqlite-utils](https://sqlite-utils.datasette.io/en/stable/python-api.html) is a Python library (and [command-line tool](https://sqlite-utils.datasette.io/en/stable/cli.html) for quickly creating and manipulating SQLite database files.\n", + "\n", + "This tutorial will show you how to use the Python library to manipulate data.\n", + "\n", + "## Installation\n", + "\n", + "To install the library, run:\n", + "\n", + " pip install sqlite-utils\n", + "\n", + "You can run this in a Jupyter notebook cell by executing:\n", + "\n", + " %pip install sqlite-utils\n", + " \n", + "Or use `pip install -U sqlite-utils` to ensure you have upgraded to the most recent version." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "bddee0d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: sqlite_utils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (3.14)\n", + "Requirement already satisfied: click-default-group in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.2.2)\n", + "Requirement already satisfied: sqlite-fts4 in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (1.0.1)\n", + "Requirement already satisfied: click in /Users/simon/Library/Python/3.9/lib/python/site-packages (from sqlite_utils) (7.1.2)\n", + "Requirement already satisfied: tabulate in /usr/local/lib/python3.9/site-packages (from sqlite_utils) (0.8.7)\n", + "Requirement already satisfied: dateutils in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from sqlite_utils) (0.6.12)\n", + "Requirement already satisfied: python-dateutil in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2.8.1)\n", + "Requirement already satisfied: pytz in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from dateutils->sqlite_utils) (2021.1)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages (from python-dateutil->dateutils->sqlite_utils) (1.16.0)\n", + "\u001b[33mWARNING: You are using pip version 21.1.1; however, version 21.2.2 is available.\n", + "You should consider upgrading via the '/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -U sqlite_utils" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "050e85a8", + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite_utils" + ] + }, + { + "cell_type": "markdown", + "id": "348bcbfc", + "metadata": {}, + "source": [ + "You can use the library with a database file on disk by running:\n", + "\n", + " db = sqlite_utils.Database(\"path/to/my/database.db\")\n", + "\n", + "In this tutorial we will use an in-memory database. This is a quick way to try out new things, though you should note that when you close the notebook the data store in the in-memory database will be lost." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4b2aee7e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ">" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite_utils.Database(memory=True)\n", + "db" + ] + }, + { + "cell_type": "markdown", + "id": "1598ab43", + "metadata": {}, + "source": [ + "## Creating a table\n", + "\n", + "We are going to create a new table in our database called `creatures` by passing in a Python list of dictionaries.\n", + "\n", + "`db[name_of_table]` will access a database table object with that name.\n", + "\n", + "Inserting data into that table will create it if it does not already exist." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "4a0ac420", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}])" + ] + }, + { + "cell_type": "markdown", + "id": "049d110b", + "metadata": {}, + "source": [ + "Let's grab a `table` reference to the new creatures table:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8d84ad9c", + "metadata": {}, + "outputs": [], + "source": [ + "table = db[\"creatures\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ffe45750", + "metadata": {}, + "source": [ + "`sqlite-utils` automatically creates a table schema that matches the keys and data types of the dictionaries that were passed to `.insert_all()`.\n", + "\n", + "We can see that schema using `table.schema`:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "136cee1e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "9e5c3ae9", + "metadata": {}, + "source": [ + "## Accessing data\n", + "\n", + "The `table.rows` property lets us loop through the rows in the table, returning each one as a Python dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f812914d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "{'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "{'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for row in table.rows:\n", + " print(row)" + ] + }, + { + "cell_type": "markdown", + "id": "60bc6b2c", + "metadata": {}, + "source": [ + "The `db.query(sql)` method can be used to execute SQL queries and return the results as dictionaries:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "eaadd85f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "6614467b", + "metadata": {}, + "source": [ + "Or in a loop:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "88fdd52e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cleo is a dog\n", + "Lila is a chicken\n", + "Bants is a chicken\n" + ] + } + ], + "source": [ + "for row in db.query(\"select name, species from creatures\"):\n", + " print(f'{row[\"name\"]} is a {row[\"species\"]}')" + ] + }, + { + "cell_type": "markdown", + "id": "b81c031c", + "metadata": {}, + "source": [ + "### SQL parameters\n", + "\n", + "You can run a parameterized query using `?` as placeholders and passing a list of variables. The variables you pass will be correctly quoted, protecting your code from SQL injection vulnerabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "267035d9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Cleo', 'species': 'dog', 'age': 6.0}]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where age > ?\", [1.0]))" + ] + }, + { + "cell_type": "markdown", + "id": "87cb301b", + "metadata": {}, + "source": [ + "As an alternative to question marks we can use `:name` parameters and feed in the values using a dictionary:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "83be9a80", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where species = :species\", {\"species\": \"chicken\"}))" + ] + }, + { + "cell_type": "markdown", + "id": "5e5179cc", + "metadata": {}, + "source": [ + "### Primary keys\n", + "\n", + "When we created this table we did not specify a primary key. SQLite automatically creates a primary key called `rowid` if no other primary key is defined.\n", + "\n", + "We can run `select rowid, * from creatures` to see this hidden primary key:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "c9d963df", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select rowid, * from creatures\"))" + ] + }, + { + "cell_type": "markdown", + "id": "0f87cdfb", + "metadata": {}, + "source": [ + "We can also see that using `table.pks_and_rows_where()`:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "d365e405", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 {'rowid': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0}\n", + "2 {'rowid': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8}\n", + "3 {'rowid': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8}\n" + ] + } + ], + "source": [ + "for pk, row in table.pks_and_rows_where():\n", + " print(pk, row)" + ] + }, + { + "cell_type": "markdown", + "id": "5b0e9b74", + "metadata": {}, + "source": [ + "Let's recreate the table with our own primary key, which we will call `id`.\n", + "\n", + "`table.drop()` drops the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "568a0e29", + "metadata": {}, + "outputs": [], + "source": [ + "table.drop()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "13ebd3ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table" + ] + }, + { + "cell_type": "markdown", + "id": "522aa6d0", + "metadata": {}, + "source": [ + "We can see a list of tables in the database using `db.tables`:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "f3e62678", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "6b80d523", + "metadata": {}, + "source": [ + "We'll create the table again, this time with an `id` column.\n", + "\n", + "We use `pk=\"id\"` to specify that the `id` column should be treated as the primary key for the table:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c9ee8b9f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db[\"creatures\"].insert_all([{\n", + " \"id\": 1,\n", + " \"name\": \"Cleo\",\n", + " \"species\": \"dog\",\n", + " \"age\": 6\n", + "}, {\n", + " \"id\": 2,\n", + " \"name\": \"Lila\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 3,\n", + " \"name\": \"Bants\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "523e01ab", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [creatures] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species] TEXT,\n", + " [age] FLOAT\n", + ")\n" + ] + } + ], + "source": [ + "print(table.schema)" + ] + }, + { + "cell_type": "markdown", + "id": "811bea70", + "metadata": {}, + "source": [ + "## Inserting more records\n", + "\n", + "We can call `.insert_all()` again to insert more records. Let's add two more chickens." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "716df161", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert_all([{\n", + " \"id\": 4,\n", + " \"name\": \"Azi\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.8,\n", + "}, {\n", + " \"id\": 5,\n", + " \"name\": \"Snowy\",\n", + " \"species\": \"chicken\",\n", + " \"age\": 0.9,\n", + "}], pk=\"id\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "4b1b2476", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "2af4ae75", + "metadata": {}, + "source": [ + "Since the `id` column is an integer primary key, we can insert a record without specifying an ID and one will be automatically added.\n", + "\n", + "Since we are only adding one record we will use `.insert()` instead of `.insert_all()`." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "246c6dd5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"name\": \"Blue\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "d7c28e4d", + "metadata": {}, + "source": [ + "We can use `table.last_pk` to see the ID of the record we just added." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "de012e1e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.last_pk" + ] + }, + { + "cell_type": "markdown", + "id": "c38edaf4", + "metadata": {}, + "source": [ + "Here's the full list of rows again:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "7c27075e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "64931bd0", + "metadata": {}, + "source": [ + "If you try to add a new record with an existing ID, you will get an `IntegrityError`:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "36327794", + "metadata": {}, + "outputs": [ + { + "ename": "IntegrityError", + "evalue": "UNIQUE constraint failed: creatures.id", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIntegrityError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"id\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"name\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"Red\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"species\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"chicken\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"age\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.9\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert\u001b[0;34m(self, record, pk, foreign_keys, column_order, not_null, defaults, hash_id, alter, ignore, replace, extracts, conversions, columns)\u001b[0m\n\u001b[1;32m 2027\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDEFAULT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2028\u001b[0m ):\n\u001b[0;32m-> 2029\u001b[0;31m return self.insert_all(\n\u001b[0m\u001b[1;32m 2030\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mrecord\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2031\u001b[0m \u001b[0mpk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_all\u001b[0;34m(self, records, pk, foreign_keys, column_order, not_null, defaults, batch_size, hash_id, alter, ignore, replace, truncate, extracts, conversions, columns, upsert)\u001b[0m\n\u001b[1;32m 2143\u001b[0m \u001b[0mfirst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2145\u001b[0;31m self.insert_chunk(\n\u001b[0m\u001b[1;32m 2146\u001b[0m \u001b[0malter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2147\u001b[0m \u001b[0mextracts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36minsert_chunk\u001b[0;34m(self, alter, extracts, chunk, all_columns, hash_id, upsert, pk, conversions, num_records_processed, replace, ignore)\u001b[0m\n\u001b[1;32m 1955\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mqueries_and_params\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1956\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1957\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1958\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mOperationalError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1959\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0malter\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\" column\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/Cellar/jupyterlab/3.0.16_1/libexec/lib/python3.9/site-packages/sqlite_utils/db.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, sql, parameters)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tracer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mparameters\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIntegrityError\u001b[0m: UNIQUE constraint failed: creatures.id" + ] + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9})" + ] + }, + { + "cell_type": "markdown", + "id": "2e00692f", + "metadata": {}, + "source": [ + "You can use `replace=True` to replace the matching record with a new one:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "2be75589", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.insert({\"id\": 6, \"name\": \"Red\", \"species\": \"chicken\", \"age\": 0.9}, replace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "83281675", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'species': 'dog', 'age': 6.0},\n", + " {'id': 2, 'name': 'Lila', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 3, 'name': 'Bants', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 4, 'name': 'Azi', 'species': 'chicken', 'age': 0.8},\n", + " {'id': 5, 'name': 'Snowy', 'species': 'chicken', 'age': 0.9},\n", + " {'id': 6, 'name': 'Red', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(table.rows)" + ] + }, + { + "cell_type": "markdown", + "id": "d7122b76", + "metadata": {}, + "source": [ + "## Updating a record\n", + "\n", + "We will rename that row back to `Blue`, this time using the `table.update(pk, updates)` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "43df156d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.update(6, {\"name\": \"Blue\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "0b8f8422", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 6, 'name': 'Blue', 'species': 'chicken', 'age': 0.9}]" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"select * from creatures where id = ?\", [6]))" + ] + }, + { + "cell_type": "markdown", + "id": "58142b86", + "metadata": {}, + "source": [ + "## Extracting one of the columns into another table\n", + "\n", + "Our current table has a `species` column with a string in it - let's pull that out into a separate table.\n", + "\n", + "We can do that using the [table.extract() method](https://sqlite-utils.datasette.io/en/stable/python-api.html#extracting-columns-into-a-separate-table)." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "6ab69111", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table.extract(\"species\")" + ] + }, + { + "cell_type": "markdown", + "id": "dca327b2", + "metadata": {}, + "source": [ + "We now have a new table called `species`, which we can see using the `db.tables` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "76e95b36", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[
,
]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.tables" + ] + }, + { + "cell_type": "markdown", + "id": "5ea43bf5", + "metadata": {}, + "source": [ + "Our creatures table has been modified - instead of a `species` column it now has `species_id` which is a foreign key to the new table:" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "c0438bff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE \"creatures\" (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [name] TEXT,\n", + " [species_id] INTEGER,\n", + " [age] FLOAT,\n", + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n", + ")\n", + "[{'id': 1, 'name': 'Cleo', 'species_id': 1, 'age': 6.0}, {'id': 2, 'name': 'Lila', 'species_id': 2, 'age': 0.8}, {'id': 3, 'name': 'Bants', 'species_id': 2, 'age': 0.8}, {'id': 4, 'name': 'Azi', 'species_id': 2, 'age': 0.8}, {'id': 5, 'name': 'Snowy', 'species_id': 2, 'age': 0.9}, {'id': 6, 'name': 'Blue', 'species_id': 2, 'age': 0.9}]\n" + ] + } + ], + "source": [ + "print(db[\"creatures\"].schema)\n", + "print(list(db[\"creatures\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "0452c201", + "metadata": {}, + "source": [ + "The new `species` table has been created and populated too:" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "5d38c3a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE [species] (\n", + " [id] INTEGER PRIMARY KEY,\n", + " [species] TEXT\n", + ")\n", + "[{'id': 1, 'species': 'dog'}, {'id': 2, 'species': 'chicken'}]\n" + ] + } + ], + "source": [ + "print(db[\"species\"].schema)\n", + "print(list(db[\"species\"].rows))" + ] + }, + { + "cell_type": "markdown", + "id": "a0312d1e", + "metadata": {}, + "source": [ + "We can use a join SQL query to combine data from these two tables:" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6734ed5d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': 1, 'name': 'Cleo', 'age': 6.0, 'species_id': 1, 'species': 'dog'},\n", + " {'id': 2, 'name': 'Lila', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 3, 'name': 'Bants', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 4, 'name': 'Azi', 'age': 0.8, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 5, 'name': 'Snowy', 'age': 0.9, 'species_id': 2, 'species': 'chicken'},\n", + " {'id': 6, 'name': 'Blue', 'age': 0.9, 'species_id': 2, 'species': 'chicken'}]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(db.query(\"\"\"\n", + " select\n", + " creatures.id,\n", + " creatures.name,\n", + " creatures.age,\n", + " species.id as species_id,\n", + " species.species\n", + " from creatures\n", + " join species on creatures.species_id = species.id\n", + "\"\"\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c4802ac", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..7a88d6f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +extend-ignore = E203 # for Black diff --git a/setup.py b/setup.py index 6e7f978..65fe6f7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.6" +VERSION = "3.15.1" def get_long_description(): @@ -22,11 +22,19 @@ setup( version=VERSION, license="Apache License, Version 2.0", packages=find_packages(exclude=["tests", "tests.*"]), - install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"], + install_requires=[ + "sqlite-fts4", + "click", + "click-default-group", + "tabulate", + "dateutils", + ], setup_requires=["pytest-runner"], extras_require={ "test": ["pytest", "black", "hypothesis"], - "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], + "mypy": ["mypy", "types-click", "types-tabulate", "types-python-dateutil"], + "flake8": ["flake8"], }, entry_points=""" [console_scripts] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e42be83..d94229d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,12 +1,14 @@ import base64 import click -from click_default_group import DefaultGroup +from click_default_group import DefaultGroup # type: ignore from datetime import datetime import hashlib import pathlib import sqlite_utils -from sqlite_utils.db import AlterError +from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils import recipes import textwrap +import inspect import io import itertools import json @@ -14,7 +16,18 @@ import os import sys import csv as csv_std import tabulate -from .utils import file_progress, find_spatialite, sqlite3, decode_base64_values +from .utils import ( + file_progress, + find_spatialite, + sqlite3, + decode_base64_values, + progressbar, + rows_from_file, + Format, + TypeTracker, +) + +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "BLOB") @@ -31,7 +44,7 @@ It's often worth trying: --encoding=latin-1 """.strip() -# Increase CSV field size limit to maximim possible +# Increase CSV field size limit to maximum possible # https://stackoverflow.com/a/15063941 field_size_limit = sys.maxsize @@ -89,7 +102,12 @@ def load_extension_option(fn): )(fn) -@click.group(cls=DefaultGroup, default="query", default_if_no_args=True) +@click.group( + cls=DefaultGroup, + default="query", + default_if_no_args=True, + context_settings=CONTEXT_SETTINGS, +) @click.version_option() def cli(): "Commands for interacting with a SQLite database" @@ -245,17 +263,6 @@ def views( ) -@cli.command() -@click.argument( - "path", - type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), - required=True, -) -def vacuum(path): - """Run VACUUM against the database""" - sqlite_utils.Database(path).vacuum() - - @cli.command() @click.argument( "path", @@ -308,6 +315,21 @@ def vacuum(path): sqlite_utils.Database(path).vacuum() +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@load_extension_option +def dump(path, load_extension): + """Output a SQL dump of the schema and full contents of the database""" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + for line in db.conn.iterdump(): + click.echo(line) + + @cli.command(name="add-column") @click.argument( "path", @@ -450,11 +472,21 @@ def index_foreign_keys(path, load_extension): ) @load_extension_option def create_index(path, table, column, name, unique, if_not_exists, load_extension): - "Add an index to the specified table covering the specified columns" + """ + Add an index to the specified table covering the specified columns. + Use "sqlite-utils create-index mydb -- -column" to specify descending + order for a column. + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) + # Treat -prefix as descending for columns + columns = [] + for col in column: + if col.startswith("-"): + col = DescIndex(col[1:]) + columns.append(col) db[table].create_index( - column, index_name=name, unique=unique, if_not_exists=if_not_exists + columns, index_name=name, unique=unique, if_not_exists=if_not_exists ) @@ -611,6 +643,7 @@ def insert_upsert_options(fn): "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("--flatten", is_flag=True, help="Flatten nested JSON objects"), click.option("-c", "--csv", is_flag=True, help="Expect CSV"), click.option("--tsv", is_flag=True, help="Expect TSV"), click.option("--delimiter", help="Delimiter to use for CSV files"), @@ -644,6 +677,13 @@ def insert_upsert_options(fn): "--encoding", help="Character encoding for input, defaults to utf-8", ), + click.option( + "-d", + "--detect-types", + is_flag=True, + envvar="SQLITE_UTILS_DETECT_TYPES", + help="Detect types for columns in CSV/TSV data", + ), load_extension_option, click.option("--silent", is_flag=True, help="Do not show progress bar"), ) @@ -658,6 +698,7 @@ def insert_upsert_implementation( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -673,6 +714,7 @@ def insert_upsert_implementation( not_null=None, default=None, encoding=None, + detect_types=None, load_extension=None, silent=False, ): @@ -682,13 +724,16 @@ def insert_upsert_implementation( csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") + if (csv or tsv) and flatten: + raise click.ClickException("--flatten cannot be used with --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") - encoding = encoding or "utf-8" - buffered = io.BufferedReader(json_file, buffer_size=4096) - decoded = io.TextIOWrapper(buffered, encoding=encoding) if pk and len(pk) == 1: pk = pk[0] + encoding = encoding or "utf-8-sig" + buffered = io.BufferedReader(json_file, buffer_size=4096) + decoded = io.TextIOWrapper(buffered, encoding=encoding) + tracker = None if csv or tsv: if sniff: # Read first 2048 bytes and use that to detect @@ -710,6 +755,9 @@ def insert_upsert_implementation( else: headers = first_row docs = (dict(zip(headers, row)) for row in reader) + if detect_types: + tracker = TypeTracker() + docs = tracker.wrap(docs) else: try: if nl: @@ -722,6 +770,8 @@ def insert_upsert_implementation( raise click.ClickException( "Invalid JSON - use --csv for CSV or --tsv for TSV files" ) + if flatten: + docs = (dict(_flatten(doc)) for doc in docs) extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} if not_null: @@ -732,9 +782,52 @@ def insert_upsert_implementation( extra_kwargs["upsert"] = upsert # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) - db[table].insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) + try: + db[table].insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except Exception as e: + if ( + isinstance(e, sqlite3.OperationalError) + and e.args + and "has no column named" in e.args[0] + ): + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format(e.args[0]) + ) + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] + ) + ) + else: + raise + if tracker is not None: + db[table].transform(types=tracker.types) + + +def _flatten(d): + for key, value in d.items(): + if isinstance(value, dict): + for key2, value2 in _flatten(value): + yield key + "_" + key2, value2 + else: + yield key, value + + +def _find_variables(tb, vars): + to_find = list(vars) + found = {} + for var in to_find: + if var in tb.tb_frame.f_locals: + vars.remove(var) + found[var] = tb.tb_frame.f_locals[var] + if vars and tb.tb_next: + found.update(_find_variables(tb.tb_next, vars)) + return found @cli.command() @@ -760,6 +853,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -769,6 +863,7 @@ def insert( batch_size, alter, encoding, + detect_types, load_extension, silent, ignore, @@ -790,6 +885,7 @@ def insert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -803,6 +899,7 @@ def insert( replace=replace, truncate=truncate, encoding=encoding, + detect_types=detect_types, load_extension=load_extension, silent=silent, not_null=not_null, @@ -820,6 +917,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, batch_size, @@ -831,6 +929,7 @@ def upsert( not_null, default, encoding, + detect_types, load_extension, silent, ): @@ -846,6 +945,7 @@ def upsert( json_file, pk, nl, + flatten, csv, tsv, delimiter, @@ -905,7 +1005,17 @@ def upsert( def create_table( path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension ): - "Add an index to the specified table covering the specified columns" + """ + Add a table with the specified columns. Columns should be specified using + name, type pairs, for example: + + \b + sqlite-utils create-table my.db people \\ + id integer \\ + name text \\ + height float \\ + photo blob --pk id + """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) if len(columns) % 2 == 1: @@ -1060,8 +1170,168 @@ def query( db.attach(alias, attach_path) _load_extensions(db, load_extension) db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +@cli.command() +@click.argument( + "paths", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + required=False, + nargs=-1, +) +@click.argument("sql") +@click.option( + "--attach", + type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), + multiple=True, + help="Additional databases to attach - specify alias and filepath", +) +@output_options +@click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for SQL query", +) +@click.option( + "--encoding", + help="Character encoding for CSV input, defaults to utf-8", +) +@click.option( + "-n", + "--no-detect-types", + is_flag=True, + help="Treat all CSV/TSV columns as TEXT", +) +@click.option("--schema", is_flag=True, help="Show SQL schema for in-memory database") +@click.option("--dump", is_flag=True, help="Dump SQL for in-memory database") +@click.option( + "--save", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + help="Save in-memory database to this file", +) +@load_extension_option +def memory( + paths, + sql, + attach, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + raw, + param, + encoding, + no_detect_types, + schema, + dump, + save, + load_extension, +): + """Execute SQL query against an in-memory database, optionally populated by imported data + + To import data from CSV, TSV or JSON files pass them on the command-line: + + \b + sqlite-utils memory one.csv two.json \\ + "select * from one join two on one.two_id = two.id" + + For data piped into the tool from standard input, use "-" or "stdin": + + \b + cat animals.csv | sqlite-utils memory - \\ + "select * from stdin where species = 'dog'" + + The format of the data will be automatically detected. You can specify the format + explicitly using :json, :csv, :tsv or :nl (for newline-delimited JSON) - for example: + + \b + cat animals.csv | sqlite-utils memory stdin:csv places.dat:nl \\ + "select * from stdin where place_id in (select id from places)" + + Use --schema to view the SQL schema of any imported files: + + \b + sqlite-utils memory animals.csv --schema + """ + db = sqlite_utils.Database(memory=True) + # If --dump or --save used but no paths detected, assume SQL query is a path: + if (dump or save or schema) and not paths: + paths = [sql] + sql = None + for i, path in enumerate(paths): + # Path may have a :format suffix + if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__: + path, suffix = path.rsplit(":", 1) + format = Format[suffix.upper()] + else: + format = None + if path in ("-", "stdin"): + csv_fp = sys.stdin.buffer + csv_table = "stdin" + else: + csv_path = pathlib.Path(path) + csv_table = csv_path.stem + csv_fp = csv_path.open("rb") + rows, format_used = rows_from_file(csv_fp, format=format, encoding=encoding) + tracker = None + if format_used in (Format.CSV, Format.TSV) and not no_detect_types: + tracker = TypeTracker() + rows = tracker.wrap(rows) + db[csv_table].insert_all(rows, alter=True) + if tracker is not None: + db[csv_table].transform(types=tracker.types) + # Add convenient t / t1 / t2 views + view_names = ["t{}".format(i + 1)] + if i == 0: + view_names.append("t") + for view_name in view_names: + if not db[view_name].exists(): + db.create_view(view_name, "select * from [{}]".format(csv_table)) + + if dump: + for line in db.conn.iterdump(): + click.echo(line) + return + + if schema: + click.echo(db.schema) + return + + if save: + db2 = sqlite_utils.Database(save) + for line in db.conn.iterdump(): + db2.execute(line) + return + + for alias, attach_path in attach: + db.attach(alias, attach_path) + _load_extensions(db, load_extension) + db.register_fts4_bm25() + + _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols + ) + + +def _execute_query( + db, sql, param, raw, table, csv, tsv, no_headers, fmt, nl, arrays, json_cols +): with db.conn: - cursor = db.execute(sql, dict(param)) + try: + cursor = db.execute(sql, dict(param)) + except sqlite3.OperationalError as e: + raise click.ClickException(str(e)) if cursor.description is None: # This was an update/insert headers = ["rows_affected"] @@ -1260,6 +1530,90 @@ def triggers( ) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +@click.option("--aux", is_flag=True, help="Include auxiliary columns") +@output_options +@load_extension_option +@click.pass_context +def indexes( + ctx, + path, + tables, + aux, + nl, + arrays, + csv, + tsv, + no_headers, + table, + fmt, + json_cols, + load_extension, +): + "Show indexes for this database" + sql = """ + select + sqlite_master.name as "table", + indexes.name as index_name, + xinfo.* + from sqlite_master + join pragma_index_list(sqlite_master.name) indexes + join pragma_index_xinfo(index_name) xinfo + where + sqlite_master.type = 'table' + """ + if tables: + quote = sqlite_utils.Database(memory=True).quote + sql += " and sqlite_master.name in ({})".format( + ", ".join(quote(table) for table in tables) + ) + if not aux: + sql += " and xinfo.key = 1" + ctx.invoke( + query, + path=path, + sql=sql, + nl=nl, + arrays=arrays, + csv=csv, + tsv=tsv, + no_headers=no_headers, + table=table, + fmt=fmt, + json_cols=json_cols, + load_extension=load_extension, + ) + + +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1, required=False) +@load_extension_option +def schema( + path, + tables, + load_extension, +): + "Show full schema for this database or for specified tables" + db = sqlite_utils.Database(path) + _load_extensions(db, load_extension) + if tables: + for table in tables: + click.echo(db[table].schema) + else: + click.echo(db.schema) + + @cli.command() @click.argument( "path", @@ -1269,9 +1623,12 @@ def triggers( @click.argument("table") @click.option( "--type", - type=(str, str), + type=( + str, + click.Choice(["INTEGER", "TEXT", "FLOAT", "BLOB"], case_sensitive=False), + ), multiple=True, - help="Change column type to X", + help="Change column type to INTEGER, TEXT, FLOAT or BLOB", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( @@ -1432,9 +1789,20 @@ def extract( @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") @click.option("--name", type=str, help="File name to use") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( - path, table, file_or_dir, column, pk, alter, replace, upsert, name, load_extension + path, + table, + file_or_dir, + column, + pk, + alter, + replace, + upsert, + name, + silent, + load_extension, ): """ Insert one or more files using BLOB columns in the specified table @@ -1471,7 +1839,7 @@ def insert_files( # Load all paths so we can show a progress bar paths_and_relative_paths = list(yield_paths_and_relative_paths()) - with click.progressbar(paths_and_relative_paths) as bar: + with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): for path, relative_path in bar: @@ -1593,6 +1961,159 @@ def analyze_tables( click.echo(details) +def _generate_convert_help(): + help = textwrap.dedent( + """ + Convert columns using Python code you supply. For example: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + '"\\n".join(textwrap.wrap(value, 10))' \\ + --import=textwrap + + "value" is a variable with the column value to be converted. + + The following common operations are available as recipe functions: + """ + ).strip() + recipe_names = [ + n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + ] + for name in recipe_names: + fn = getattr(recipes, name) + help += "\n\nr.{}{}\n\n {}".format( + name, str(inspect.signature(fn)), fn.__doc__ + ) + help += "\n\n" + help += textwrap.dedent( + """ + You can use these recipes like so: + + \b + $ sqlite-utils convert my.db mytable mycolumn \\ + 'r.jsonsplit(value, delimiter=":")' + """ + ).strip() + return help + + +@cli.command(help=_generate_convert_help()) +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table", type=str) +@click.argument("columns", type=str, nargs=-1, required=True) +@click.argument("code", type=str) +@click.option( + "--import", "imports", type=str, multiple=True, help="Python modules to import" +) +@click.option( + "--dry-run", is_flag=True, help="Show results of running this against first 10 rows" +) +@click.option( + "--multi", is_flag=True, help="Populate columns for keys in returned dictionary" +) +@click.option("--where", help="Optional where clause") +@click.option( + "-p", + "--param", + multiple=True, + type=(str, str), + help="Named :parameters for where clause", +) +@click.option("--output", help="Optional separate column to populate with the output") +@click.option( + "--output-type", + help="Column type to use for the output column", + default="text", + type=click.Choice(["integer", "float", "blob", "text"]), +) +@click.option("--drop", is_flag=True, help="Drop original column afterwards") +@click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") +def convert( + db_path, + table, + columns, + code, + imports, + dry_run, + multi, + where, + param, + output, + output_type, + drop, + silent, +): + sqlite3.enable_callback_tracebacks(True) + db = sqlite_utils.Database(db_path) + if output is not None and len(columns) > 1: + raise click.ClickException("Cannot use --output with more than one column") + if multi and len(columns) > 1: + raise click.ClickException("Cannot use --multi with more than one column") + if drop and not (output or multi): + raise click.ClickException("--drop can only be used with --output or --multi") + # If single line and no 'return', add the return + if "\n" not in code and not code.strip().startswith("return "): + code = "return {}".format(code) + where_args = dict(param) if param else [] + # Compile the code into a function body called fn(value) + new_code = ["def fn(value):"] + for line in code.split("\n"): + new_code.append(" {}".format(line)) + code_o = compile("\n".join(new_code), "", "exec") + locals = {} + globals = {"r": recipes, "recipes": recipes} + for import_ in imports: + globals[import_] = __import__(import_) + exec(code_o, globals, locals) + fn = locals["fn"] + if dry_run: + # Pull first 20 values for first column and preview them + db.conn.create_function("preview_transform", 1, lambda v: fn(v) if v else v) + sql = """ + select + [{column}] as value, + preview_transform([{column}]) as preview + from [{table}]{where} limit 10 + """.format( + column=columns[0], + table=table, + where=" where {}".format(where) if where is not None else "", + ) + for row in db.conn.execute(sql, where_args).fetchall(): + click.echo(str(row[0])) + click.echo(" --- becomes:") + click.echo(str(row[1])) + click.echo() + count = db[table].count_where( + where=where, + where_args=where_args, + ) + click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s")) + else: + try: + db[table].convert( + columns, + fn, + where=where, + where_args=where_args, + output=output, + output_type=output_type, + drop=drop, + multi=multi, + show_progress=not silent, + ) + except BadMultiValues as e: + raise click.ClickException( + "When using --multi code must return a Python dictionary - returned: {}".format( + repr(e.values) + ) + ) + + def _render_common(title, values): if values is None: return "" diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7c59c9c..f51dba6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,12 @@ -from .utils import sqlite3, OperationalError, suggest_column_types, column_affinity -from collections import namedtuple, OrderedDict +from .utils import ( + sqlite3, + OperationalError, + suggest_column_types, + types_for_column_types, + column_affinity, + progressbar, +) +from collections import namedtuple from collections.abc import Mapping import contextlib import datetime @@ -11,9 +18,22 @@ import json import os import pathlib import re -from sqlite_fts4 import rank_bm25 +from sqlite_fts4 import rank_bm25 # type: ignore import sys import textwrap +from typing import ( + cast, + Any, + Callable, + Dict, + Generator, + Iterable, + Union, + Optional, + List, + Set, + Tuple, +) import uuid SQLITE_MAX_VARS = 999 @@ -35,24 +55,46 @@ _virtual_table_using_re = re.compile( ) ) \s+(IF\s+NOT\s+EXISTS\s+)? # IF NOT EXISTS (optional) -USING\s+(?P\w+) # e.g. USING FTS5 +USING\s+(?P\w+) # for example USING FTS5 """, re.VERBOSE | re.IGNORECASE, ) try: - import pandas as pd + import pandas as pd # type: ignore except ImportError: - pd = None + pd = None # type: ignore try: - import numpy as np + import numpy as np # type: ignore except ImportError: - np = None + np = None # type: ignore Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) +Column.__doc__ = """ +Describes a SQLite column returned by the :attr:`.Table.columns` property. + +``cid`` + Column index + +``name`` + Column name + +``type`` + Column type + +``notnull`` + Does the column have a ``not null`` constraint + +``default_value`` + Default value for this column + +``is_pk`` + Is this column part of the primary key +""" + ColumnDetails = namedtuple( "ColumnDetails", ( @@ -66,14 +108,50 @@ ColumnDetails = namedtuple( "least_common", ), ) +ColumnDetails.__doc__ = """ +Summary information about a column, see :ref:`python_api_analyze_column`. + +``table`` + The name of the table + +``column`` + The name of the column + +``total_rows`` + The total number of rows in the table + +``num_null`` + The number of rows for which this column is null + +``num_blank`` + The number of rows for which this column is blank (the empty string) + +``num_distinct`` + The number of distinct values in this column + +``most_common`` + The ``N`` most common values as a list of ``(value, count)`` tuples, or ``None`` if the table consists entirely of distinct values + +``least_common`` + The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct + or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) +""" ForeignKey = namedtuple( "ForeignKey", ("table", "column", "other_table", "other_column") ) Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) +XIndex = namedtuple("XIndex", ("name", "columns")) +XIndexColumn = namedtuple( + "XIndexColumn", ("seqno", "cid", "name", "desc", "coll", "key") +) Trigger = namedtuple("Trigger", ("name", "table", "sql")) -DEFAULT = object() +class Default: + pass + + +DEFAULT = Default() COLUMN_TYPE_MAPPING = { float: "FLOAT", @@ -123,29 +201,46 @@ if pd: class AlterError(Exception): + "Error altering table" pass class NoObviousTable(Exception): + "Could not tell which table this operation refers to" pass class BadPrimaryKey(Exception): + "Table does not have a single obvious primary key" pass class NotFoundError(Exception): + "Record not found" pass class PrimaryKeyRequired(Exception): + "Primary key needs to be specified" pass class InvalidColumns(Exception): + "Specified columns do not exist" pass +class DescIndex(str): + pass + + +class BadMultiValues(Exception): + "With multi=True code must return a Python dictionary" + + def __init__(self, values): + self.values = values + + _COUNTS_TABLE_CREATE_SQL = """ CREATE TABLE IF NOT EXISTS [{}]( [table] TEXT PRIMARY KEY, @@ -155,17 +250,39 @@ CREATE TABLE IF NOT EXISTS [{}]( class Database: + """ + Wrapper for a SQLite database connection that adds a variety of useful utility methods. + + To create an instance:: + + # create data.db file, or open existing: + db = Database("data.db") + # Create an in-memory database: + dB = Database(memory=True) + + - ``filename_or_conn`` - String path to a file, or a ``pathlib.Path`` object, or a + ``sqlite3`` connection + - ``memory`` - set to ``True`` to create an in-memory database + - ``recreate`` - set to ``True`` to delete and recreate a file database (**dangerous**) + - ``recursive_triggers`` - defaults to ``True``, which sets ``PRAGMA recursive_triggers=on;`` - + set to ``False`` to avoid setting this pragma + - ``tracer`` - set a tracer function (``print`` works for this) which will be called with + ``sql, parameters`` every time a SQL query is executed + - ``use_counts_table`` - set to ``True`` to use a cached counts table, if available. See + :ref:`python_api_cached_table_counts`. + """ + _counts_table_name = "_counts" use_counts_table = False def __init__( self, filename_or_conn=None, - memory=False, - recreate=False, - recursive_triggers=True, - tracer=None, - use_counts_table=False, + memory: bool = False, + recreate: bool = False, + recursive_triggers: bool = True, + tracer: Callable = None, + use_counts_table: bool = False, ): assert (filename_or_conn is not None and not memory) or ( filename_or_conn is None and memory @@ -182,11 +299,24 @@ class Database: self._tracer = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") - self._registered_functions = set() + self._registered_functions: set = set() self.use_counts_table = use_counts_table @contextlib.contextmanager - def tracer(self, tracer=None): + def tracer(self, tracer: Callable = None): + """ + Context manager to temporarily set a tracer function - all executed SQL queries will + be passed to this. + + The tracer function should accept two arguments: ``sql`` and ``parameters`` + + Example usage:: + + with db.tracer(print): + db["creatures"].insert({"name": "Cleo"}) + + See :ref:`python_api_tracing`. + """ prev_tracer = self._tracer self._tracer = tracer or print try: @@ -194,13 +324,39 @@ class Database: finally: self._tracer = prev_tracer - def __getitem__(self, table_name): + def __getitem__(self, table_name: str) -> Union["Table", "View"]: + """ + ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. + If the table does not exist yet it will be created the first time data is inserted into it. + """ return self.table(table_name) def __repr__(self): return "".format(self.conn) - def register_function(self, fn=None, deterministic=None, replace=False): + def register_function( + self, fn: Callable = None, deterministic: bool = False, replace: bool = False + ): + """ + ``fn`` will be made available as a function within SQL, with the same name and number + of arguments. Can be used as a decorator:: + + @db.register + def upper(value): + return str(value).upper() + + The decorator can take arguments:: + + @db.register(deterministic=True, replace=True) + def upper(value): + return str(value).upper() + + - ``deterministic`` - set ``True`` for functions that always returns the same output for a given input + - ``replace`` - set ``True`` to replace an existing function with the same name - otherwise throw an error + + See :ref:`python_api_register_function`. + """ + def register(fn): name = fn.__name__ arity = len(inspect.signature(fn).parameters) @@ -219,9 +375,15 @@ class Database: register(fn) def register_fts4_bm25(self): + "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True) - def attach(self, alias, filepath): + def attach(self, alias: str, filepath: Union[str, pathlib.Path]): + """ + Attach another SQLite database file to this connection with the specified alias, equivalent to:: + + ATTACH DATABASE 'filepath.db' AS alias + """ attach_sql = """ ATTACH DATABASE '{}' AS [{}]; """.format( @@ -229,7 +391,19 @@ class Database: ).strip() self.execute(attach_sql) - def execute(self, sql, parameters=None): + def query( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> Generator[dict, None, None]: + "Execute ``sql`` and return an iterable of dictionaries representing each row." + cursor = self.execute(sql, params or tuple()) + keys = [d[0] for d in cursor.description] + for row in cursor: + yield dict(zip(keys, row)) + + def execute( + self, sql: str, parameters: Optional[Union[Iterable, dict]] = None + ) -> sqlite3.Cursor: + "Execute SQL query and return a ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, parameters) if parameters is not None: @@ -237,16 +411,19 @@ class Database: else: return self.conn.execute(sql) - def executescript(self, sql): + def executescript(self, sql: str) -> sqlite3.Cursor: + "Execute multiple SQL statements separated by ; and return the ``sqlite3.Cursor``." if self._tracer: self._tracer(sql, None) return self.conn.executescript(sql) - def table(self, table_name, **kwargs): + def table(self, table_name: str, **kwargs) -> Union["Table", "View"]: + "Return a table object, optionally configured with default options." klass = View if table_name in self.view_names() else Table return klass(self, table_name, **kwargs) - def quote(self, value): + def quote(self, value: str) -> str: + "Apply SQLite string quoting to a value, including wrappping it in single quotes." # Normally we would use .execute(sql, [params]) for escaping, but # occasionally that isn't available - most notable when we need # to include a "... DEFAULT 'value'" in a column definition. @@ -256,8 +433,8 @@ class Database: {"value": value}, ).fetchone()[0] - - def quote_fts(self, query): + def quote_fts(self, query: str) -> str: + "Escape special characters in a SQLite full-text search query" # NOTE: This is not a query validator for FTS. Sqlite has # a well defined query syntax here: # https://www2.sqlite.org/fts5.html#full_text_query_syntax @@ -275,7 +452,8 @@ class Database: '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits ) - def table_names(self, fts4=False, fts5=False): + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + "A list of string table names in this database." where = ["type = 'table'"] if fts4: where.append("sql like '%USING FTS4%'") @@ -284,7 +462,8 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self): + def view_names(self) -> List[str]: + "A list of string view names in this database." return [ r[0] for r in self.execute( @@ -293,15 +472,18 @@ class Database: ] @property - def tables(self): - return [self[name] for name in self.table_names()] + def tables(self) -> List["Table"]: + "A list of Table objects in this database." + return cast(List["Table"], [self[name] for name in self.table_names()]) @property - def views(self): - return [self[name] for name in self.view_names()] + def views(self) -> List["View"]: + "A list of View objects in this database." + return cast(List["View"], [self[name] for name in self.view_names()]) @property - def triggers(self): + def triggers(self) -> List[Trigger]: + "A list of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) for r in self.execute( @@ -310,19 +492,35 @@ class Database: ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def journal_mode(self): + def schema(self) -> str: + "SQL schema for this database" + sqls = [] + for row in self.execute( + "select sql from sqlite_master where sql is not null" + ).fetchall(): + sql = row[0] + if not sql.strip().endswith(";"): + sql += ";" + sqls.append(sql) + return "\n".join(sqls) + + @property + def journal_mode(self) -> str: + "Current ``journal_mode`` of this database." return self.execute("PRAGMA journal_mode;").fetchone()[0] def enable_wal(self): + "Set ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode." if self.journal_mode != "wal": self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): + "Set ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." if self.journal_mode != "delete": self.execute("PRAGMA journal_mode=delete;") @@ -331,6 +529,10 @@ class Database: self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) def enable_counts(self): + """ + Enable trigger-based count caching for every table in the database, see + :ref:`python_api_cached_table_counts`. + """ self._ensure_counts_table() for table in self.tables: if ( @@ -340,7 +542,11 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables=None): + def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + """ + Return ``{table_name: count}`` dictionary of cached counts for specified tables, or + all tables if ``tables`` not provided. + """ sql = "select [table], count from {}".format(self._counts_table_name) if tables: sql += " where [table] in ({})".format(", ".join("?" for table in tables)) @@ -350,6 +556,7 @@ class Database: return {} def reset_counts(self): + "Re-calculate cached counts for tables." tables = [table for table in self.tables if table.has_counts_triggers] with self.conn: self._ensure_counts_table() @@ -360,10 +567,10 @@ class Database: for table in tables ) - def execute_returning_dicts(self, sql, params=None): - cursor = self.execute(sql, params or tuple()) - keys = [d[0] for d in cursor.description] - return [dict(zip(keys, row)) for row in cursor.fetchall()] + def execute_returning_dicts( + self, sql: str, params: Optional[Union[Iterable, dict]] = None + ) -> List[dict]: + return list(self.query(sql, params)) def resolve_foreign_keys(self, name, foreign_keys): # foreign_keys may be a list of strcolumn names, a list of ForeignKey tuples, @@ -408,16 +615,17 @@ class Database: def create_table_sql( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> str: + "Returns the SQL ``CREATE TABLE`` statement for creating the specified table." foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} # any extracts will be treated as integer columns with a foreign key @@ -514,16 +722,21 @@ class Database: def create_table( self, - name, - columns, - pk=None, + name: str, + columns: Dict[str, Any], + pk: Optional[Any] = None, foreign_keys=None, column_order=None, not_null=None, defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified name and the specified ``{column_name: type}`` columns. + + See :ref:`python_api_explicit_create`. + """ sql = self.create_table_sql( name=name, columns=columns, @@ -536,7 +749,7 @@ class Database: extracts=extracts, ) self.execute(sql) - return self.table( + table = self.table( name, pk=pk, foreign_keys=foreign_keys, @@ -545,8 +758,17 @@ class Database: defaults=defaults, hash_id=hash_id, ) + return cast(Table, table) - def create_view(self, name, sql, ignore=False, replace=False): + def create_view( + self, name: str, sql: str, ignore: bool = False, replace: bool = False + ): + """ + Create a new SQL view with the specified name - ``sql`` should start with ``SELECT ...``. + + - ``ignore`` - set to ``True`` to do nothing if a view with this name already exists + - ``replace`` - set to ``True`` to replace the view if one with this name already exists + """ assert not ( ignore and replace ), "Use one or the other of ignore/replace, not both" @@ -564,18 +786,28 @@ class Database: self.execute(create_sql) return self - def m2m_table_candidates(self, table, other_table): - "Returns potential m2m tables for arguments, based on FKs" + def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + """ + Given two table names returns the name of tables that could define a + many-to-many relationship between those two tables, based on having + foreign keys to both of the provided tables. + """ candidates = [] tables = {table, other_table} - for table in self.tables: + for table_obj in self.tables: # Does it have foreign keys to both table and other_table? - has_fks_to = {fk.other_table for fk in table.foreign_keys} + has_fks_to = {fk.other_table for fk in table_obj.foreign_keys} if has_fks_to.issuperset(tables): - candidates.append(table.name) + candidates.append(table_obj.name) return candidates - def add_foreign_keys(self, foreign_keys): + def add_foreign_keys(self, foreign_keys: Iterable[Tuple[str, str, str, str]]): + """ + See :ref:`python_api_add_foreign_keys`. + + ``foreign_keys`` should be a list of ``(table, column, other_table, other_column)`` + tuples, see :ref:`python_api_add_foreign_keys`. + """ # foreign_keys is a list of explicit 4-tuples assert all( len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys @@ -587,7 +819,11 @@ class Database: for table, column, other_table, other_column in foreign_keys: if not self[table].exists(): raise AlterError("No such table: {}".format(table)) - if column not in self[table].columns_dict: + table_obj = self[table] + if not isinstance(table_obj, Table): + raise AlterError("Must be a table, not a view: {}".format(table)) + table_obj = cast(Table, table_obj) + if column not in table_obj.columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) @@ -601,7 +837,7 @@ class Database: # We will silently skip foreign keys that exist already if not any( fk - for fk in self[table].foreign_keys + for fk in table_obj.foreign_keys if fk.column == column and fk.other_table == other_table and fk.other_column == other_column @@ -611,7 +847,7 @@ class Database: ) # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - table_sql = {} + table_sql: Dict[str, str] = {} for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY([{column}]) REFERENCES [{other_table}]([{other_column}])\n".format( @@ -639,6 +875,7 @@ class Database: self.vacuum() def index_foreign_keys(self): + "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self[table_name] existing_indexes = { @@ -649,41 +886,67 @@ class Database: table.create_index([fk.column]) def vacuum(self): + "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") class Queryable: - def exists(self): + def exists(self) -> bool: + "Does this table or view exist yet?" return False def __init__(self, db, name): self.db = db self.name = name + def count_where( + self, + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + ) -> int: + "Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count." + sql = "select count(*) from [{}]".format(self.name) + if where is not None: + sql += " where " + where + return self.db.execute(sql, where_args or []).fetchone()[0] + def execute_count(self): - return self.db.execute( - "select count(*) from [{}]".format(self.name) - ).fetchone()[0] + # Backwards compatibility, see https://github.com/simonw/sqlite-utils/issues/305#issuecomment-890713185 + return self.count_where() @property - def count(self): - return self.execute_count() + def count(self) -> int: + "A count of the rows in this table or view." + return self.count_where() @property - def rows(self): + def rows(self) -> Generator[dict, None, None]: + "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where=None, - where_args=None, - order_by=None, - select="*", - limit=None, - offset=None, - ): + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + select: str = "*", + limit: int = None, + offset: int = None, + ) -> Generator[dict, None, None]: + """ + Iterate over every row in this table or view that matches the specified where clause. + + - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``order_by`` - optional column or fragment of SQL to order by. + - ``select`` - optional comma-separated list of columns to select. + - ``limit`` - optional integer number of rows to limit to. + - ``offset`` - optional integer for SQL offset. + + Returns each row as a dictionary. See :ref:`python_api_rows` for more details. + """ if not self.exists(): - return [] + return sql = "select {} from [{}]".format(select, self.name) if where is not None: sql += " where " + where @@ -700,13 +963,13 @@ class Queryable: def pks_and_rows_where( self, - where=None, - where_args=None, - order_by=None, - limit=None, - offset=None, - ): - "Like .rows_where() but returns (pk, row) pairs - pk can be a single value or tuple" + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + order_by: str = None, + limit: int = None, + offset: int = None, + ) -> Generator[Tuple[Any, Dict], None, None]: + "Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple." column_names = [column.name for column in self.columns] pks = [column.name for column in self.columns if column.is_pk] if not pks: @@ -727,32 +990,37 @@ class Queryable: yield row_pk, row @property - def columns(self): + def columns(self) -> List["Column"]: + "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute("PRAGMA table_info([{}])".format(self.name)).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self): - "Returns {column: python-type} dictionary" + def columns_dict(self) -> Dict[str, Any]: + "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @property - def schema(self): + def schema(self) -> str: + "SQL schema for this table or view." return self.db.execute( "select sql from sqlite_master where name = ?", (self.name,) ).fetchone()[0] class Table(Queryable): - last_rowid = None - last_pk = None + "Tables should usually be initialized using the ``db.table(table_name)`` or ``db[table_name]`` methods." + #: The ``rowid`` of the last inserted, updated or selected row. + last_rowid: Optional[int] = None + #: The primary key of the last inserted, updated or selected row. + last_pk: Optional[Any] = None def __init__( self, - db, - name, + db: Database, + name: str, pk=None, foreign_keys=None, column_order=None, @@ -784,7 +1052,7 @@ class Table(Queryable): columns=columns, ) - def __repr__(self): + def __repr__(self) -> str: return "
".format( self.name, " (does not exist yet)" @@ -793,24 +1061,38 @@ class Table(Queryable): ) @property - def count(self): + def count(self) -> int: + "Count of the rows in this table - optionally from the table count cache, if configured." if self.db.use_counts_table: counts = self.db.cached_counts([self.name]) if counts: return next(iter(counts.values())) - return self.execute_count() + return self.count_where() def exists(self): return self.name in self.db.table_names() @property - def pks(self): + def pks(self) -> List[str]: + "Primary key columns for this table." names = [column.name for column in self.columns if column.is_pk] if not names: names = ["rowid"] return names - def get(self, pk_values): + @property + def use_rowid(self) -> bool: + "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" + return not any(column for column in self.columns if column.is_pk) + + def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + """ + Return row (as dictionary) for the specified primary key. + + Primary key can be a single value, or a tuple for tables with a compound primary key. + + Raises ``NotFoundError`` if a matching row cannot be found. + """ if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] pks = self.pks @@ -832,7 +1114,8 @@ class Table(Queryable): raise NotFoundError @property - def foreign_keys(self): + def foreign_keys(self) -> List["ForeignKey"]: + "List of foreign keys defined on this table." fks = [] for row in self.db.execute( "PRAGMA foreign_key_list([{}])".format(self.name) @@ -850,15 +1133,16 @@ class Table(Queryable): return fks @property - def virtual_table_using(self): - "Returns type of virtual table or None if this is not a virtual table" + def virtual_table_using(self) -> Optional[str]: + "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: return None return match.groupdict()["using"].upper() @property - def indexes(self): + def indexes(self) -> List[Index]: + "List of indexes defined on this table." sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): @@ -881,7 +1165,27 @@ class Table(Queryable): return indexes @property - def triggers(self): + def xindexes(self) -> List[XIndex]: + "List of indexes defined on this table using the more detailed ``XIndex`` format." + sql = 'PRAGMA index_list("{}")'.format(self.name) + indexes = [] + for row in self.db.execute_returning_dicts(sql): + index_name = row["name"] + index_name_quoted = ( + '"{}"'.format(index_name) + if not index_name.startswith('"') + else index_name + ) + column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) + index_columns = [] + for info in self.db.execute(column_sql).fetchall(): + index_columns.append(XIndexColumn(*info)) + indexes.append(XIndex(index_name, index_columns)) + return indexes + + @property + def triggers(self) -> List[Trigger]: + "List of triggers defined on this table." return [ Trigger(*r) for r in self.db.execute( @@ -892,8 +1196,8 @@ class Table(Queryable): ] @property - def triggers_dict(self): - "Returns {trigger_name: sql} dictionary" + def triggers_dict(self) -> Dict[str, str]: + "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} def create( @@ -906,7 +1210,12 @@ class Table(Queryable): defaults=None, hash_id=None, extracts=None, - ): + ) -> "Table": + """ + Create a table with the specified columns. + + See :ref:`python_api_explicit_create` for full details. + """ columns = {name: value for (name, value) in columns.items()} with self.db.conn: self.db.create_table( @@ -933,7 +1242,13 @@ class Table(Queryable): defaults=None, drop_foreign_keys=None, column_order=None, - ): + ) -> "Table": + """ + Apply an advanced alter table, including operations that are not supported by + ``ALTER TABLE`` in SQLite itself. + + See :ref:`python_api_transform` for full details. + """ assert self.exists(), "Cannot transform a table that doesn't exist yet" sqls = self.transform_sql( types=types, @@ -974,7 +1289,8 @@ class Table(Queryable): drop_foreign_keys=None, column_order=None, tmp_suffix=None, - ): + ) -> List[str]: + "Returns a list of SQL statements that would be executed in order to apply this transformation." types = types or {} rename = rename or {} drop = drop or set() @@ -995,7 +1311,9 @@ class Table(Queryable): sqls = [] if pk is DEFAULT: - pks_renamed = tuple(rename.get(p) or p for p in self.pks) + pks_renamed = tuple( + rename.get(p.name) or p.name for p in self.columns if p.is_pk + ) if len(pks_renamed) == 1: pk = pks_renamed[0] else: @@ -1078,7 +1396,18 @@ class Table(Queryable): ) return sqls - def extract(self, columns, table=None, fk_column=None, rename=None): + def extract( + self, + columns: Union[str, Iterable[str]], + table: Optional[str] = None, + fk_column: Optional[str] = None, + rename: Optional[Dict[str, str]] = None, + ) -> "Table": + """ + Extract specified columns into a separate table. + + See :ref:`python_api_extract` for details. + """ rename = rename or {} if isinstance(columns, str): columns = [columns] @@ -1089,8 +1418,6 @@ class Table(Queryable): ) ) table = table or "_".join(columns) - first_column = columns[0] - pks = self.pks lookup_table = self.db[table] fk_column = fk_column or "{}_id".format(table) magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) @@ -1172,11 +1499,35 @@ class Table(Queryable): self.add_foreign_key(fk_column, table, "id") return self - def create_index(self, columns, index_name=None, unique=False, if_not_exists=False): + def create_index( + self, + columns: Iterable[Union[str, DescIndex]], + index_name: Optional[str] = None, + unique: bool = False, + if_not_exists: bool = False, + ): + """ + Create an index on this table. + + - ``columns`` - a single columns or list of columns to index. These can be strings or, + to create an index using the column in descending order, ``db.DescIndex(column_name)`` objects. + - ``index_name`` - the name to use for the new index. Defaults to the column names joined on ``_``. + - ``unique`` - should the index be marked as unique, forcing unique values? + - ``if_not_exists`` - only create the index if one with that name does not already exist. + + See :ref:`python_api_create_index`. + """ if index_name is None: index_name = "idx_{}_{}".format( self.name.replace(" ", "_"), "_".join(columns) ) + columns_sql = [] + for column in columns: + if isinstance(column, DescIndex): + fmt = "[{}] desc" + else: + fmt = "[{}]" + columns_sql.append(fmt.format(column)) sql = ( textwrap.dedent( """ @@ -1188,7 +1539,7 @@ class Table(Queryable): .format( index_name=index_name, table_name=self.name, - columns=", ".join("[{}]".format(c) for c in columns), + columns=", ".join(columns_sql), unique="UNIQUE " if unique else "", if_not_exists="IF NOT EXISTS " if if_not_exists else "", ) @@ -1197,12 +1548,13 @@ class Table(Queryable): return self def add_column( - self, col_name, col_type=None, fk=None, fk_col=None, not_null_default=None + self, col_name: str, col_type=None, fk=None, fk_col=None, not_null_default=None ): + "Add a column to this table. See :ref:`python_api_add_column`." fk_col_type = None if fk is not None: # fk must be a valid table - if not fk in self.db.table_names(): + if fk not in self.db.table_names(): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: @@ -1233,14 +1585,24 @@ class Table(Queryable): self.add_foreign_key(col_name, fk, fk_col) return self - def drop(self, ignore=False): + def drop(self, ignore: bool = False): + "Drop this table. ``ignore=True`` means errors will be ignored." try: self.db.execute("DROP TABLE [{}]".format(self.name)) except sqlite3.OperationalError: if not ignore: raise - def guess_foreign_table(self, column): + def guess_foreign_table(self, column: str) -> str: + """ + For a given column, suggest another table that might be referenced by this + column should it be used as a foreign key. + + For example, a column called ``tag_id`` or ``tag`` or ``tags`` might suggest + a ``tag`` table, if one exists. + + If no candidates can be found, raises a ``NoObviousTable`` exception. + """ column = column.lower() possibilities = [column] if column.endswith("_id"): @@ -1261,7 +1623,7 @@ class Table(Queryable): ) ) - def guess_foreign_column(self, other_table): + def guess_foreign_column(self, other_table: str): pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( @@ -1271,8 +1633,20 @@ class Table(Queryable): return pks[0].name def add_foreign_key( - self, column, other_table=None, other_column=None, ignore=False + self, + column: str, + other_table: Optional[str] = None, + other_column: Optional[str] = None, + ignore: bool = False, ): + """ + Alter the schema to mark the specified column as a foreign key to another table. + + - ``column`` - the column to mark as a foreign key. + - ``other_table`` - the table it refers to - if omitted, will be guessed based on the column name. + - ``other_column`` - the column on the other table it - if omitted, will be guessed. + - ``ignore`` - set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. + """ # Ensure column exists if column not in self.columns_dict: raise AlterError("No such column: {}".format(column)) @@ -1309,6 +1683,11 @@ class Table(Queryable): return self def enable_counts(self): + """ + Set up triggers to update a cache of the count of rows in this table. + + See :ref:`python_api_cached_table_counts` for details. + """ sql = ( textwrap.dedent( """ @@ -1353,7 +1732,8 @@ class Table(Queryable): self.db.use_counts_table = True @property - def has_counts_triggers(self): + def has_counts_triggers(self) -> bool: + "Does this table have triggers setup to update cached counts?" trigger_names = { "{table}{counts_table}_{suffix}".format( counts_table=self.db._counts_table_name, table=self.name, suffix=suffix @@ -1364,13 +1744,23 @@ class Table(Queryable): def enable_fts( self, - columns, - fts_version="FTS5", - create_triggers=False, - tokenize=None, - replace=False, + columns: Iterable[str], + fts_version: str = "FTS5", + create_triggers: bool = False, + tokenize: Optional[str] = None, + replace: bool = False, ): - "Enables FTS on the specified columns." + """ + Enable SQLite full-text search against the specified columns. + + - ``columns`` - list of column names to include in the search index. + - ``fts_version`` - FTS version to use - defaults to ``FTS5`` but you may want ``FTS4`` for older SQLite versions. + - ``create_triggers`` - should triggers be created to keep the search index up-to-date? Defaults to ``False``. + - ``tokenize`` - custom SQLite tokenizer to use, for example ``"porter"`` to enable Porter stemming. + - ``replace`` - should any existing FTS index for this table be replaced by the new one? + + See :ref:`python_api_fts` for more details. + """ create_fts_sql = ( textwrap.dedent( """ @@ -1438,7 +1828,11 @@ class Table(Queryable): self.db.executescript(triggers) return self - def populate_fts(self, columns): + def populate_fts(self, columns: Iterable[str]) -> "Table": + """ + Update the associated SQLite full-text search index with the latest data from the + table for the specified columns. + """ sql = ( textwrap.dedent( """ @@ -1454,7 +1848,8 @@ class Table(Queryable): self.db.executescript(sql) return self - def disable_fts(self): + def disable_fts(self) -> "Table": + "Remove any full-text search index and related triggers configured for this table." fts_table = self.detect_fts() if fts_table: self.db[fts_table].drop() @@ -1479,6 +1874,7 @@ class Table(Queryable): return self def rebuild_fts(self): + "Run the ``rebuild`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is None: # Assume this is itself an FTS table @@ -1490,7 +1886,7 @@ class Table(Queryable): ) return self - def detect_fts(self): + def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" sql = ( textwrap.dedent( @@ -1515,7 +1911,8 @@ class Table(Queryable): else: return rows[0][0] - def optimize(self): + def optimize(self) -> "Table": + "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: self.db.execute( @@ -1527,7 +1924,8 @@ class Table(Queryable): ) return self - def search_sql(self, columns=None, order_by=None, limit=None, offset=None): + def search_sql(self, columns=None, order_by=None, limit=None, offset=None) -> str: + "Return SQL string that can be used to execute searches against this table." # Pick names for table and rank column that don't clash original = "original_" if self.name == "original" else "original" columns_sql = "*" @@ -1584,7 +1982,26 @@ class Table(Queryable): limit_offset=limit_offset.strip(), ).strip() - def search(self, q, order_by=None, columns=None, limit=None, offset=None): + def search( + self, + q: str, + order_by: Optional[str] = None, + columns: Optional[List[str]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[dict, None, None]: + """ + Execute a search against this table using SQLite full-text search, returning a sequence of + dictionaries for each row. + + - ``q`` - words to search for + - ``order_by`` - defaults to order by rank, or specify a column here. + - ``columns`` - list of columns to return, defaults to all columns. + - ``limit`` - optional integer limit for returned rows. + - ``offset`` - optional integer SQL offset. + + See :ref:`python_api_fts_search`. + """ cursor = self.db.execute( self.search_sql( order_by=order_by, @@ -1601,7 +2018,8 @@ class Table(Queryable): def value_or_default(self, key, value): return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values): + def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + "Delete row matching the specified primary key." if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) @@ -1613,16 +2031,37 @@ class Table(Queryable): self.db.execute(sql, pk_values) return self - def delete_where(self, where=None, where_args=None): + def delete_where( + self, where: str = None, where_args: Optional[Union[Iterable, dict]] = None + ) -> "Table": + "Delete rows matching specified where clause, or delete all rows in the table." if not self.exists(): - return [] + return self sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) return self - def update(self, pk_values, updates=None, alter=False, conversions=None): + def update( + self, + pk_values: Union[list, tuple, str, int, float], + updates: Optional[dict] = None, + alter: bool = False, + conversions: Optional[dict] = None, + ) -> "Table": + """ + Execute a SQL ``UPDATE`` against the specified row. + + - ``pk_values`` - the primary key of an individual record - can be a tuple if the + table has a compound primary key. + - ``updates`` - a dictionary mapping columns to their updated values. + - ``alter`` - set to ``True`` to add any missing columns. + - ``conversions`` - optional dictionary of SQL functions to apply during the update, for example + ``{"mycolumn": "upper(?)"}``. + + See :ref:`python_api_update`. + """ updates = updates or {} conversions = conversions or {} if not isinstance(pk_values, (list, tuple)): @@ -1660,6 +2099,131 @@ class Table(Queryable): self.last_pk = pk_values[0] if len(pks) == 1 else pk_values return self + def convert( + self, + columns: Union[str, List[str]], + fn: Callable, + output: Optional[str] = None, + output_type: Optional[Any] = None, + drop: bool = False, + multi: bool = False, + where: Optional[str] = None, + where_args: Optional[Union[Iterable, dict]] = None, + show_progress: bool = False, + ): + """ + Apply conversion function ``fn`` to every value in the specified columns. + + - ``columns`` - a single column or list of string column names to convert. + - ``fn`` - a callable that takes a single argument, ``value``, and returns it converted. + - ``output`` - optional string column name to write the results to (defaults to the input column). + - ``output_type`` - if the output column needs to be created, this is the type that will be used + for the new column. + - ``drop`` - boolean, should the original column be dropped once the conversion is complete? + - ``multi`` - boolean, if ``True`` the return value of ``fn(value)`` will be expected to be a + dictionary, and new columns will be created for each key of that dictionary. + - ``where`` - a SQL fragment to use as a ``WHERE`` clause to limit the rows to which the conversion + is applied, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``show_progress`` - boolean, should a progress bar be displayed? + + See :ref:`python_api_convert`. + """ + if isinstance(columns, str): + columns = [columns] + + if multi: + return self._convert_multi( + columns[0], + fn, + drop=drop, + where=where, + where_args=where_args, + show_progress=show_progress, + ) + + if output is not None: + assert len(columns) == 1, "output= can only be used with a single column" + if output not in self.columns_dict: + self.add_column(output, output_type or "text") + + todo_count = self.count_where(where, where_args) * len(columns) + with progressbar(length=todo_count, silent=not show_progress) as bar: + + def convert_value(v): + bar.update(1) + if not v: + return v + return fn(v) + + self.db.register_function(convert_value) + sql = "update [{table}] set {sets}{where};".format( + table=self.name, + sets=", ".join( + [ + "[{output_column}] = convert_value([{column}])".format( + output_column=output or column, column=column + ) + for column in columns + ] + ), + where=" where {}".format(where) if where is not None else "", + ) + with self.db.conn: + self.db.execute(sql, where_args or []) + if drop: + self.transform(drop=columns) + return self + + def _convert_multi( + self, column, fn, drop, show_progress, where=None, where_args=None + ): + # First we execute the function + pk_to_values = {} + new_column_types = {} + pks = [column.name for column in self.columns if column.is_pk] + if not pks: + pks = ["rowid"] + + with progressbar( + length=self.count, silent=not show_progress, label="1: Evaluating" + ) as bar: + for row in self.rows_where( + select=", ".join( + "[{}]".format(column_name) for column_name in (pks + [column]) + ), + where=where, + where_args=where_args, + ): + row_pk = tuple(row[pk] for pk in pks) + if len(row_pk) == 1: + row_pk = row_pk[0] + values = fn(row[column]) + if values is not None and not isinstance(values, dict): + raise BadMultiValues(values) + if values: + for key, value in values.items(): + new_column_types.setdefault(key, set()).add(type(value)) + pk_to_values[row_pk] = values + bar.update(1) + + # Add any new columns + columns_to_create = types_for_column_types(new_column_types) + for column_name, column_type in columns_to_create.items(): + if column_name not in self.columns_dict: + self.add_column(column_name, column_type) + + # Run the updates + with progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar: + with self.db.conn: + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) + def build_insert_queries_and_params( self, extracts, @@ -1708,20 +2272,22 @@ class Table(Queryable): queries_and_params.append((sql, [record[col] for col in pks])) # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; set_cols = [col for col in all_columns if col not in pks] - sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( - table=self.name, - pairs=", ".join( - "[{}] = {}".format(col, conversions.get(col, "?")) - for col in set_cols - ), - wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), - ) - queries_and_params.append( - ( - sql2, - [record[col] for col in set_cols] + [record[pk] for pk in pks], + if set_cols: + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join( + "[{}] = {}".format(col, conversions.get(col, "?")) + for col in set_cols + ), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + + [record[pk] for pk in pks], + ) ) - ) # We can populate .last_pk right here if num_records_processed == 1: self.last_pk = tuple(record[pk] for pk in pks) @@ -1843,20 +2409,51 @@ class Table(Queryable): def insert( self, - record, + record: Dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - ): + column_order: Optional[Union[List[str], Default]] = DEFAULT, + not_null: Optional[Union[Set[str], Default]] = DEFAULT, + defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + hash_id: Optional[Union[str, Default]] = DEFAULT, + alter: Optional[Union[bool, Default]] = DEFAULT, + ignore: Optional[Union[bool, Default]] = DEFAULT, + replace: Optional[Union[bool, Default]] = DEFAULT, + extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, + conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, + columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + ) -> "Table": + """ + Insert a single record into the table. The table will be created with a schema that matches + the inserted record if it does not already exist, see :ref:`python_api_creating_tables`. + + - ``record`` - required: a dictionary representing the record to be inserted. + + The other parameters are optional, and mostly influence how the new table will be created if + that table does not exist yet. + + Each of them defaults to ``DEFAULT``, which indicates that the default setting for the current + ``Table`` object (specified in the table constructor) should be used. + + - ``pk`` - if creating the table, which column should be the primary key. + - ``foreign_keys`` - see :ref:`python_api_foreign_keys`. + - ``column_order`` - optional list of strings specifying a full or partial column order + to use when creating the table. + - ``not_null`` - optional set of strings specifying columns that should be ``NOT NULL``. + - ``defaults`` - optional dictionary specifying default values for specific columns. + - ``hash_id`` - optional name of a column to create and use as a primary key, where the + value of thet primary key will be derived as a SHA1 hash of the other column values + in the record. ``hash_id="id"`` is a common column name used for this. + - ``alter`` - boolean, should any missing columns be added automatically? + - ``ignore`` - boolean, if a record already exists with this primary key, ignore this insert. + - ``replace`` - boolean, if a record already exists with this primary key, replace it with this new record. + - ``extracts`` - a list of columns to extract to other tables, or a dictionary that maps + ``{column_name: other_table_name}``. See :ref:`python_api_extracts`. + - ``conversions`` - dictionary specifying SQL conversion functions to be applied to the data while it + is being inserted, for example ``{"name": "upper(?)"}``. See :ref:`python_api_conversions`. + - ``columns`` - dictionary over-riding the detected types used for the columns, for example + ``{"age": int, "weight": float}``. + """ return self.insert_all( [record], pk=pk, @@ -1891,11 +2488,10 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, - ): + ) -> "Table": """ - Like .insert() but takes a list of records and ensures that the table - that it creates (if table does not exist) has columns for ALL of that - data + Like ``.insert()`` but takes a list of records and ensures that the table + that it creates (if table does not exist) has columns for ALL of that data. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -1920,7 +2516,7 @@ class Table(Queryable): assert not ( ignore and replace ), "Use either ignore=True or replace=True, not both" - all_columns = None + all_columns = [] first = True num_records_processed = 0 # We can only handle a max of 999 variables in a SQL insert, so @@ -1958,10 +2554,10 @@ class Table(Queryable): hash_id=hash_id, extracts=extracts, ) - all_columns = set() + all_columns_set = set() for record in chunk: - all_columns.update(record.keys()) - all_columns = list(sorted(all_columns)) + all_columns_set.update(record.keys()) + all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) else: @@ -2002,7 +2598,13 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do + not exist and updated if they DO exist, based on matching against their primary key. + + See :ref:`python_api_upsert`. + """ return self.upsert_all( [record], pk=pk, @@ -2031,7 +2633,10 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, - ): + ) -> "Table": + """ + Like ``.upsert()`` but can be applied to a list of records. + """ return self.insert_all( records, pk=pk, @@ -2048,7 +2653,7 @@ class Table(Queryable): upsert=True, ) - def add_missing_columns(self, records): + def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -2056,7 +2661,20 @@ class Table(Queryable): self.add_column(col_name, col_type) return self - def lookup(self, column_values): + def lookup(self, column_values: Dict[str, Any]): + """ + Create or populate a lookup table with the specified values. + + ``db["Species"].lookup({"name": "Palm"})`` will create a table called ``Species`` + (if one does not already exist) with two columns: ``id`` and ``name``. It will + set up a unique constraint on the ``name`` column to guarantee it will not + contain duplicate rows. + + It well then inserts a new row with the ``name`` set to ``Palm`` and return the + new integer primary key value. + + See :ref:`python_api_lookup_tables` for more details. + """ # lookups is a dictionary - all columns will be used for a unique index assert isinstance(column_values, dict) if self.exists(): @@ -2081,15 +2699,38 @@ class Table(Queryable): def m2m( self, - other_table, - record_or_iterable=None, - pk=DEFAULT, - lookup=None, - m2m_table=None, - alter=False, + other_table: Union[str, "Table"], + record_or_iterable: Optional[ + Union[Iterable[Dict[str, Any]], Dict[str, Any]] + ] = None, + pk: Optional[Union[Any, Default]] = DEFAULT, + lookup: Optional[Dict[str, Any]] = None, + m2m_table: Optional[str] = None, + alter: bool = False, ): + """ + After inserting a record in a table, create one or more records in some other + table and then create many-to-many records linking the original record and the + newly created records together. + + For example:: + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + See :ref:`python_api_m2m` for details. + + - ``other_table`` - the name of the table to insert the new records into. + - ``record_or_iterable`` - a single dictionary record to insert, or a list of records. + - ``pk`` - the primary key to use if creating ``other_table``. + - ``lookup`` - same dictionary as for ``.lookup()``, to create a many-to-many lookup table. + - ``m2m_table`` - the string name to use for the many-to-many table, defaults to creating + this automatically based on the names of the two tables. + - ``alter`` - set to ``True`` to add any missing columns on ``other_table`` if that table + already exists. + """ if isinstance(other_table, str): - other_table = self.db.table(other_table, pk=pk) + other_table = cast(Table, self.db.table(other_table, pk=pk)) our_id = self.last_pk if lookup is not None: assert record_or_iterable is None, "Provide lookup= or record, not both" @@ -2113,20 +2754,19 @@ class Table(Queryable): else: # If not, create a new table m2m_table_name = m2m_table or "{}_{}".format(*tables) - m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) + m2m_table_obj = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: # if records is only one record, put the record in a list - records = ( - [record_or_iterable] - if isinstance(record_or_iterable, Mapping) - else record_or_iterable - ) + if isinstance(record_or_iterable, Mapping): + records = [record_or_iterable] + else: + records = cast(List, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( - record, pk=pk, replace=True, alter=alter + cast(dict, record), pk=pk, replace=True, alter=alter ).last_pk - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2135,7 +2775,7 @@ class Table(Queryable): ) else: id = other_table.lookup(lookup) - m2m_table.insert( + m2m_table_obj.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, @@ -2145,8 +2785,13 @@ class Table(Queryable): return self def analyze_column( - self, column, common_limit=10, value_truncate=None, total_rows=None - ): + self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None + ) -> "ColumnDetails": + """ + Return statistics about the specified column. + + See :ref:`python_api_analyze_column`. + """ db = self.db table = self.name if total_rows is None: @@ -2243,7 +2888,7 @@ def jsonify_if_needed(value): if isinstance(value, decimal.Decimal): return float(value) if isinstance(value, (dict, list, tuple)): - return json.dumps(value, default=repr) + return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() elif isinstance(value, uuid.UUID): diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py new file mode 100644 index 0000000..6918661 --- /dev/null +++ b/sqlite_utils/recipes.py @@ -0,0 +1,19 @@ +from dateutil import parser +import json + + +def parsedate(value, dayfirst=False, yearfirst=False): + "Parse a date and convert it to ISO date format: yyyy-mm-dd" + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() + ) + + +def parsedatetime(value, dayfirst=False, yearfirst=False): + "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + + +def jsonsplit(value, delimiter=",", type=str): + 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' + return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 9d5dac6..00a3c02 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -1,16 +1,22 @@ import base64 -import click import contextlib +import csv +import enum import io +import json import os +from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type + +import click try: - import pysqlite3 as sqlite3 - import pysqlite3.dbapi2 + import pysqlite3 as sqlite3 # type: ignore + import pysqlite3.dbapi2 # type: ignore OperationalError = pysqlite3.dbapi2.OperationalError except ImportError: - import sqlite3 + # https://github.com/python/mypy/issues/1153#issuecomment-253842414 + import sqlite3 # type: ignore OperationalError = sqlite3.OperationalError @@ -25,8 +31,11 @@ def suggest_column_types(records): for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) - column_types = {} + return types_for_column_types(all_column_types) + +def types_for_column_types(all_column_types): + column_types = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -105,9 +114,167 @@ class UpdateWrapper: @contextlib.contextmanager def file_progress(file, silent=False, **kwargs): - if silent or file.fileno() == 0: # 0 = stdin + if silent: + yield file + return + # file.fileno() throws an exception in our test suite + try: + fileno = file.fileno() + except io.UnsupportedOperation: + yield file + return + if fileno == 0: # 0 means stdin yield file else: file_length = os.path.getsize(file.name) with click.progressbar(length=file_length, **kwargs) as bar: yield UpdateWrapper(file, bar.update) + + +class Format(enum.Enum): + CSV = 1 + TSV = 2 + JSON = 3 + NL = 4 + + +class RowsFromFileError(Exception): + pass + + +class RowsFromFileBadJSON(RowsFromFileError): + pass + + +def rows_from_file( + fp: BinaryIO, + format: Optional[Format] = None, + dialect: Optional[Type[csv.Dialect]] = None, + encoding: Optional[str] = None, +) -> Tuple[Iterable[dict], Format]: + if format == Format.JSON: + decoded = json.load(fp) + if isinstance(decoded, dict): + decoded = [decoded] + if not isinstance(decoded, list): + raise RowsFromFileBadJSON("JSON must be a list or a dictionary") + return decoded, Format.JSON + elif format == Format.NL: + return (json.loads(line) for line in fp if line.strip()), Format.NL + elif format == Format.CSV: + use_encoding: str = encoding or "utf-8-sig" + decoded_fp = io.TextIOWrapper(fp, encoding=use_encoding) + if dialect is not None: + reader = csv.DictReader(decoded_fp, dialect=dialect) + else: + reader = csv.DictReader(decoded_fp) + return reader, Format.CSV + elif format == Format.TSV: + return ( + rows_from_file( + fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding + )[0], + Format.TSV, + ) + elif format is None: + # Detect the format, then call this recursively + buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096) + first_bytes = buffered.peek(2048).strip() + if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): + # TODO: Detect newline-JSON + return rows_from_file(buffered, format=Format.JSON) + else: + dialect = csv.Sniffer().sniff( + first_bytes.decode(encoding or "utf-8-sig", "ignore") + ) + return rows_from_file( + buffered, format=Format.CSV, dialect=dialect, encoding=encoding + ) + else: + raise RowsFromFileError("Bad format") + + +class TypeTracker: + def __init__(self): + self.trackers = {} + + def wrap(self, iterator): + for row in iterator: + for key, value in row.items(): + tracker = self.trackers.setdefault(key, ValueTracker()) + tracker.evaluate(value) + yield row + + @property + def types(self): + return {key: tracker.guessed_type for key, tracker in self.trackers.items()} + + +class ValueTracker: + def __init__(self): + self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} + + @classmethod + def get_tests(cls): + return [ + key.split("test_")[-1] + for key in cls.__dict__.keys() + if key.startswith("test_") + ] + + def test_integer(self, value): + try: + int(value) + return True + except (ValueError, TypeError): + return False + + def test_float(self, value): + try: + float(value) + return True + except (ValueError, TypeError): + return False + + def __repr__(self): + return self.guessed_type + ": possibilities = " + repr(self.couldbe) + + @property + def guessed_type(self): + options = set(self.couldbe.keys()) + # Return based on precedence + for key in self.get_tests(): + if key in options: + return key + return "text" + + def evaluate(self, value): + if not value or not self.couldbe: + return + not_these = [] + for name, test in self.couldbe.items(): + if not test(value): + not_these.append(name) + for key in not_these: + del self.couldbe[key] + + +class NullProgressBar: + def __init__(self, *args): + self.args = args + + def __iter__(self): + yield from self.args[0] + + def update(self, value): + pass + + +@contextlib.contextmanager +def progressbar(*args, **kwargs): + silent = kwargs.pop("silent") + if silent: + yield NullProgressBar(*args) + else: + with click.progressbar(*args, **kwargs) as bar: + yield bar diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index f0af2ca..5795a7a 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,10 +1,8 @@ -from sqlite_utils.db import Database, ForeignKey, ColumnDetails +from sqlite_utils.db import Database, ColumnDetails from sqlite_utils import cli -from sqlite_utils.utils import OperationalError from click.testing import CliRunner import pytest import sqlite3 -import textwrap @pytest.fixture @@ -132,6 +130,7 @@ def test_analyze_table_save(db_to_analyze_path): result = CliRunner().invoke( cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] ) + assert result.exit_code == 0 rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) assert rows == [ { diff --git a/tests/test_cli.py b/tests/test_cli.py index 8775396..d4801ac 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ from sqlite_utils import cli, Database from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner +from unittest import mock import json import os import pytest @@ -24,6 +25,22 @@ def db_path(tmpdir): return path +@pytest.mark.parametrize( + "options", + ( + ["-h"], + ["--help"], + ["insert", "-h"], + ["insert", "--help"], + ), +) +def test_help(options): + result = CliRunner().invoke(cli.cli, options) + assert result.exit_code == 0 + assert result.output.startswith("Usage: ") + assert "-h, --help" in result.output + + def test_tables(db_path): result = CliRunner().invoke(cli.cli, ["tables", db_path]) assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() @@ -208,6 +225,17 @@ def test_create_index(db_path): ) +def test_create_index_desc(db_path): + db = Database(db_path) + assert [] == db["Gosh"].indexes + result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"]) + assert result.exit_code == 0 + assert ( + db.execute("select sql from sqlite_master where type='index'").fetchone()[0] + == "CREATE INDEX [idx_Gosh_c1]\n ON [Gosh] ([c1] desc)" + ) + + @pytest.mark.parametrize( "col_name,col_type,expected_schema", ( @@ -354,6 +382,21 @@ def test_add_column_foreign_key(db_path): assert "table 'bobcats' does not exist" in str(result.exception) +def test_suggest_alter_if_column_missing(db_path): + db = Database(db_path) + db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "authors", "-"], + input='{"id": 2, "name": "Barry", "age": 43}', + ) + assert result.exit_code != 0 + assert result.output.strip() == ( + "Error: table authors has no column named age\n\n" + "Try using --alter to add additional columns" + ) + + def test_index_foreign_keys(db_path): test_add_column_foreign_key(db_path) db = Database(db_path) @@ -367,7 +410,7 @@ def test_index_foreign_keys(db_path): def test_enable_fts(db_path): db = Database(db_path) - assert None == db["Gosh"].detect_fts() + assert db["Gosh"].detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) @@ -377,7 +420,7 @@ def test_enable_fts(db_path): # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert None == db["http://example.com"].detect_fts() + assert db["http://example.com"].detect_fts() is None result = CliRunner().invoke( cli.cli, [ @@ -476,6 +519,13 @@ def test_vacuum(db_path): assert 0 == result.exit_code +def test_dump(db_path): + result = CliRunner().invoke(cli.cli, ["dump", db_path]) + assert result.exit_code == 0 + assert result.output.startswith("BEGIN TRANSACTION;") + assert result.output.strip().endswith("COMMIT;") + + @pytest.mark.parametrize("tables", ([], ["Gosh"], ["Gosh2"])) def test_optimize(db_path, tables): db = Database(db_path) @@ -563,8 +613,8 @@ def test_insert_simple(tmpdir): open(json_path, "w").write(json.dumps({"name": "Cleo", "age": 4})) result = CliRunner().invoke(cli.cli, ["insert", db_path, "dogs", json_path]) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) db = Database(db_path) assert ["dogs"] == db.table_names() @@ -579,8 +629,8 @@ def test_insert_from_stdin(tmpdir): input=json.dumps({"name": "Cleo", "age": 4}), ) assert 0 == result.exit_code - assert [{"age": 4, "name": "Cleo"}] == Database(db_path).execute_returning_dicts( - "select * from dogs" + assert [{"age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") ) @@ -598,6 +648,34 @@ def test_insert_invalid_json_error(tmpdir): ) +def test_insert_json_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten"], + input=json.dumps({"nested": {"data": 4}}), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [{"nested_data": 4}] + + +def test_insert_json_flatten_nl(tmpdir): + db_path = str(tmpdir / "flat.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-", "--flatten", "--nl"], + input="\n".join( + json.dumps(item) + for item in [{"nested": {"data": 4}}, {"nested": {"other": 3}}] + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from items")) == [ + {"nested_data": 4, "nested_other": None}, + {"nested_data": None, "nested_other": 3}, + ] + + def test_insert_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dog.json") open(json_path, "w").write(json.dumps({"id": 1, "name": "Cleo", "age": 4})) @@ -605,9 +683,9 @@ def test_insert_with_primary_key(db_path, tmpdir): cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] ) assert 0 == result.exit_code - assert [{"id": 1, "age": 4, "name": "Cleo"}] == Database( - db_path - ).execute_returning_dicts("select * from dogs") + assert [{"id": 1, "age": 4, "name": "Cleo"}] == list( + Database(db_path).query("select * from dogs") + ) db = Database(db_path) assert ["id"] == db["dogs"].pks @@ -621,7 +699,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by id") + assert dogs == list(db.query("select * from dogs order by id")) assert ["id"] == db["dogs"].pks @@ -637,7 +715,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): ) assert 0 == result.exit_code db = Database(db_path) - assert dogs == db.execute_returning_dicts("select * from dogs order by breed, id") + assert dogs == list(db.query("select * from dogs order by breed, id")) assert {"breed", "id"} == set(db["dogs"].pks) assert ( "CREATE TABLE [dogs] (\n" @@ -682,7 +760,7 @@ def test_insert_binary_base64(db_path): ) assert 0 == result.exit_code, result.output db = Database(db_path) - actual = db.execute_returning_dicts("select content from files") + actual = list(db.query("select content from files")) assert actual == [{"content": b"hello"}] @@ -697,7 +775,7 @@ def test_insert_newline_delimited(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_ignore(db_path, tmpdir): @@ -716,9 +794,7 @@ def test_insert_ignore(db_path, tmpdir): ) assert 0 == result.exit_code, result.output # ... but it should actually have no effect - assert [{"id": 1, "name": "Cleo"}] == db.execute_returning_dicts( - "select * from dogs" - ) + assert [{"id": 1, "name": "Cleo"}] == list(db.query("select * from dogs")) @pytest.mark.parametrize( @@ -781,8 +857,9 @@ def test_insert_replace(db_path, tmpdir): ) assert 0 == result.exit_code, result.output assert 21 == db["dogs"].count - assert insert_replace_dogs == db.execute_returning_dicts( - "select * from dogs where id in (1, 2, 21) order by id" + assert ( + list(db.query("select * from dogs where id in (1, 2, 21) order by id")) + == insert_replace_dogs ) @@ -797,7 +874,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bar", "n": 1}, {"foo": "baz", "n": 2}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) # Truncate and insert new rows result = CliRunner().invoke( cli.cli, @@ -816,7 +893,7 @@ def test_insert_truncate(db_path): assert [ {"foo": "bam", "n": 3}, {"foo": "bat", "n": 4}, - ] == db.execute_returning_dicts("select foo, n from from_json_nl") + ] == list(db.query("select foo, n from from_json_nl")) def test_insert_alter(db_path, tmpdir): @@ -847,7 +924,7 @@ def test_insert_alter(db_path, tmpdir): {"foo": "bar", "n": 1, "baz": None}, {"foo": "baz", "n": 2, "baz": None}, {"foo": "bar", "baz": 5, "n": None}, - ] == db.execute_returning_dicts("select foo, n, baz from from_json_nl") + ] == list(db.query("select foo, n, baz from from_json_nl")) @pytest.mark.parametrize( @@ -917,7 +994,23 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() -LOREM_IPSUM_COMPRESSED = b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8ef\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +LOREM_IPSUM_COMPRESSED = ( + b"x\x9c\xed\xd1\xcdq\x03!\x0c\x05\xe0\xbb\xabP\x01\x1eW\x91\xdc|M\x01\n\xc8\x8e" + b"f\xf83H\x1e\x97\x1f\x91M\x8e\xe9\xe0\xdd\x96\x05\x84\xf4\xbek\x9fRI\xc7\xf2J" + b"\xb9\x97>i\xa9\x11W\xb13\xa5\xde\x96$\x13\xf3I\x9cu\xe8J\xda\xee$EcsI\x8e\x0b" + b"$\xea\xab\xf6L&u\xc4emI\xb3foFnT\xf83\xca\x93\xd8QZ\xa8\xf2\xbd1q\xd1\x87\xf3" + b"\x85>\x8c\xa4i\x8d\xdaTu\x7f\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\x03" + b"\x1f\xf8\xc0\x07>\xf0\x81\x0f|\xe0\xfb\x8f\xef\x1b\x9b\x06\x83}" +) def test_query_json_binary(db_path): @@ -939,7 +1032,18 @@ def test_query_json_binary(db_path): "sz": 16984, "data": { "$base64": True, - "encoded": "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uIjnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3fiCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9", + "encoded": ( + ( + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" + ) + ), }, } ] @@ -1023,7 +1127,7 @@ def test_query_load_extension(use_spatialite_shortcut): # Without --load-extension: result = CliRunner().invoke(cli.cli, [":memory:", "select spatialite_version()"]) assert result.exit_code == 1 - assert "no such function: spatialite_version" in repr(result) + assert "no such function: spatialite_version" in result.output # With --load-extension: if use_spatialite_shortcut: load_extension = "spatialite" @@ -1108,17 +1212,32 @@ def test_upsert(db_path, tmpdir): {"id": 1, "age": 5}, {"id": 2, "age": 5}, ] - open(json_path, "w").write(json.dumps(insert_dogs)) + open(json_path, "w").write(json.dumps(upsert_dogs)) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"], catch_exceptions=False, ) assert 0 == result.exit_code, result.output - assert [ - {"id": 1, "name": "Cleo", "age": 4}, - {"id": 2, "name": "Nixie", "age": 4}, - ] == db.execute_returning_dicts("select * from dogs order by id") + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] + + +def test_upsert_flatten(tmpdir): + db_path = str(tmpdir / "flat.db") + db = Database(db_path) + db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"], + input=json.dumps({"id": 1, "nested": {"two": 2}}), + ) + assert result.exit_code == 0 + assert list(db.query("select * from upsert_me")) == [ + {"id": 1, "name": "Example", "nested_two": 2} + ] def test_upsert_alter(db_path, tmpdir): @@ -1137,7 +1256,11 @@ def test_upsert_alter(db_path, tmpdir): cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] ) assert 1 == result.exit_code - assert "no such column: age" == str(result.exception) + assert ( + "Error: no such column: age\n\n" + "sql = UPDATE [dogs] SET [age] = ? WHERE [id] = ?\n" + "parameters = [5, 1]" + ) == result.output.strip() # Should succeed with --alter result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] @@ -1145,7 +1268,7 @@ def test_upsert_alter(db_path, tmpdir): assert 0 == result.exit_code assert [ {"id": 1, "name": "Cleo", "age": 5}, - ] == db.execute_returning_dicts("select * from dogs order by id") + ] == list(db.query("select * from dogs order by id")) @pytest.mark.parametrize( @@ -1499,7 +1622,7 @@ def test_query_update(db_path, args, expected): cli.cli, [db_path, "update dogs set age = 5 where name = 'Cleo'"] + args ) assert expected == result.output.strip() - assert db.execute_returning_dicts("select * from dogs") == [ + assert list(db.query("select * from dogs")) == [ {"id": 1, "age": 5, "name": "Cleo"}, ] @@ -1547,47 +1670,112 @@ def test_add_foreign_keys(db_path): [ ( [], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--type", "age", "text"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] TEXT NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] TEXT NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--drop", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT\n" + ")" + ), ), ( ["--rename", "age", "age2", "--rename", "id", "pk"], - "CREATE TABLE \"dogs\" (\n [pk] INTEGER PRIMARY KEY,\n [age2] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [pk] INTEGER PRIMARY KEY,\n" + " [age2] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--not-null", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT NOT NULL\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT NOT NULL\n" + ")" + ), ), ( ["--not-null-false", "age"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--pk", "name"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT PRIMARY KEY\n" + ")" + ), ), ( ["--pk-none"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT\n" + ")" + ), ), ( ["--default", "name", "Turnip"], - "CREATE TABLE \"dogs\" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL DEFAULT '1',\n [name] TEXT DEFAULT 'Turnip'\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [name] TEXT DEFAULT 'Turnip'\n" + ")" + ), ), ( ["--default-none", "age"], - 'CREATE TABLE "dogs" (\n [id] INTEGER PRIMARY KEY,\n [age] INTEGER NOT NULL,\n [name] TEXT\n)', + ( + 'CREATE TABLE "dogs" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [age] INTEGER NOT NULL,\n" + " [name] TEXT\n" + ")" + ), ), ( ["-o", "name", "--column-order", "age", "-o", "id"], - "CREATE TABLE \"dogs\" (\n [name] TEXT,\n [age] INTEGER NOT NULL DEFAULT '1',\n [id] INTEGER PRIMARY KEY\n)", + ( + 'CREATE TABLE "dogs" (\n' + " [name] TEXT,\n" + " [age] INTEGER NOT NULL DEFAULT '1',\n" + " [id] INTEGER PRIMARY KEY\n" + ")" + ), ), ], ) @@ -1636,9 +1824,13 @@ def test_transform_drop_foreign_key(db_path): print(result.output) assert result.exit_code == 0 schema = db["places"].schema - assert ( - schema - == 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)' + assert schema == ( + 'CREATE TABLE "places" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [name] TEXT,\n" + " [country] INTEGER,\n" + " [city] INTEGER REFERENCES [city]([id])\n" + ")" ) @@ -1652,22 +1844,48 @@ _common_other_schema = ( [ ( [], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--table", "custom_table"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_table_id] INTEGER,\n" + " FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n" + ")" + ), "CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ( ["--fk-column", "custom_fk"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)', + ( + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [custom_fk] INTEGER,\n" + " FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n" + ")" + ), _common_other_schema, ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)', + 'CREATE TABLE "trees" (\n' + " [id] INTEGER PRIMARY KEY,\n" + " [address] TEXT,\n" + " [species_id] INTEGER,\n" + " FOREIGN KEY([species_id]) REFERENCES [species]([id])\n" + ")", "CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)", ), ], @@ -1776,7 +1994,87 @@ def test_search(tmpdir, fts, extra_arg, expected): assert result.output.replace("\r", "") == expected -_TRIGGERS_EXPECTED = '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +def test_indexes(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db.conn.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_idx on Gosh(c2, c3 desc); + """ + ) + result = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path)], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + ] + result2 = CliRunner().invoke( + cli.cli, + ["indexes", str(db_path), "--aux"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + assert json.loads(result2.output) == [ + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 0, + "cid": 1, + "name": "c2", + "desc": 0, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 1, + "cid": 2, + "name": "c3", + "desc": 1, + "coll": "BINARY", + "key": 1, + }, + { + "table": "Gosh", + "index_name": "Gosh_idx", + "seqno": 2, + "cid": -1, + "name": None, + "desc": 0, + "coll": "BINARY", + "key": 0, + }, + ] + + +_TRIGGERS_EXPECTED = ( + '[{"name": "blah", "table": "articles", "sql": "CREATE TRIGGER blah ' + 'AFTER INSERT ON articles\\nBEGIN\\n UPDATE counter SET count = count + 1;\\nEND"}]\n' +) @pytest.mark.parametrize( @@ -1813,6 +2111,60 @@ def test_triggers(tmpdir, extra_args, expected): assert result.output == expected +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + ( + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ");\n" + "CREATE INDEX [idx_chickens_breed]\n" + " ON [chickens] ([breed]);\n" + ), + ), + ( + ["dogs"], + ("CREATE TABLE [dogs] (\n" " [id] INTEGER,\n" " [name] TEXT\n" ")\n"), + ), + ( + ["chickens", "dogs"], + ( + "CREATE TABLE [chickens] (\n" + " [id] INTEGER,\n" + " [name] TEXT,\n" + " [breed] TEXT\n" + ")\n" + "CREATE TABLE [dogs] (\n" + " [id] INTEGER,\n" + " [name] TEXT\n" + ")\n" + ), + ), + ), +) +def test_schema(tmpdir, options, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["dogs"].create({"id": int, "name": str}) + db["chickens"].create({"id": int, "name": str, "breed": str}) + db["chickens"].create_index(["breed"]) + result = CliRunner().invoke( + cli.cli, + ["schema", db_path] + options, + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert result.output == expected + + def test_long_csv_column_value(tmpdir): db_path = str(tmpdir / "test.db") csv_path = str(tmpdir / "test.csv") @@ -1889,3 +2241,85 @@ def test_attach(tmpdir): {"id": 1, "text": "foo"}, {"id": 1, "text": "bar"}, ] + + +def test_csv_insert_bom(tmpdir): + db_path = str(tmpdir / "test.db") + bom_csv_path = str(tmpdir / "bom.csv") + with open(bom_csv_path, "wb") as fp: + fp.write(b"\xef\xbb\xbfname,age\nCleo,5") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "broken", bom_csv_path, "--encoding", "utf-8", "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + result2 = CliRunner().invoke( + cli.cli, + ["insert", db_path, "fixed", bom_csv_path, "--csv"], + catch_exceptions=False, + ) + assert result2.exit_code == 0 + db = Database(db_path) + tables = db.execute("select name, sql from sqlite_master").fetchall() + assert tables == [ + ("broken", "CREATE TABLE [broken] (\n [\ufeffname] TEXT,\n [age] TEXT\n)"), + ("fixed", "CREATE TABLE [fixed] (\n [name] TEXT,\n [age] TEXT\n)"), + ] + + +@pytest.mark.parametrize("option_or_env_var", (None, "-d", "--detect-types")) +def test_insert_detect_types(tmpdir, option_or_env_var): + db_path = str(tmpdir / "test.db") + data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" + extra = [] + if option_or_env_var: + extra = [option_or_env_var] + + def _test(): + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--csv"] + extra, + catch_exceptions=False, + input=data, + ) + assert result.exit_code == 0 + db = Database(db_path) + assert list(db["creatures"].rows) == [ + {"name": "Cleo", "age": 6, "weight": 45.5}, + {"name": "Dori", "age": 1, "weight": 3.5}, + ] + + if option_or_env_var is None: + # Use environemnt variable instead of option + with mock.patch.dict(os.environ, {"SQLITE_UTILS_DETECT_TYPES": "1"}): + _test() + else: + _test() + + +@pytest.mark.parametrize( + "input,expected", + ( + ({"foo": {"bar": 1}}, {"foo_bar": 1}), + ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), + ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), + ), +) +def test_flatten_helper(input, expected): + assert dict(cli._flatten(input)) == expected + + +def test_integer_overflow_error(tmpdir): + db_path = str(tmpdir / "test.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "items", "-"], + input=json.dumps({"bignumber": 34223049823094832094802398430298048240}), + ) + assert result.exit_code == 1 + assert result.output == ( + "Error: Python int too large to convert to SQLite INTEGER\n\n" + "sql = INSERT INTO [items] ([bignumber]) VALUES (?);\n" + "parameters = [34223049823094832094802398430298048240]\n" + ) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py new file mode 100644 index 0000000..8ce5203 --- /dev/null +++ b/tests/test_cli_convert.py @@ -0,0 +1,517 @@ +from click.testing import CliRunner +from sqlite_utils import cli +import sqlite_utils +import json +import textwrap +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return db, db_path + + +@pytest.fixture +def fresh_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + return db, db_path + + +@pytest.mark.parametrize( + "code", + [ + "return value.replace('October', 'Spooktober')", + # Return is optional: + "value.replace('October', 'Spooktober')", + ], +) +def test_convert_single_line(test_db_and_path, code): + db, db_path = test_db_and_path + result = CliRunner().invoke(cli.cli, ["convert", db_path, "example", "dt", code]) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th Spooktober 2019 12:04"}, + {"id": 2, "dt": "6th Spooktober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_multiple_lines(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "v = value.replace('October', 'Spooktober')\nreturn v.upper()", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5TH SPOOKTOBER 2019 12:04"}, + {"id": 2, "dt": "6TH SPOOKTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_import(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + ], + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "dt": "5th OXXober 2019 12:04"}, + {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] == list(db["example"].rows) + + +def test_convert_dryrun(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + ], + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "5th October 2019 12:04\n" + " --- becomes:\n" + "5th OXXober 2019 12:04\n" + "\n" + "6th October 2019 00:05:06\n" + " --- becomes:\n" + "6th OXXober 2019 00:05:06\n" + "\n" + "\n" + " --- becomes:\n" + "\n" + "\n" + "None\n" + " --- becomes:\n" + "None\n\n" + "Would affect 4 rows" + ) + # But it should not have actually modified the table data + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + # Test with a where clause too + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "return re.sub('O..', 'OXX', value)", + "--import", + "re", + "--dry-run", + "--where", + "id = :id", + "-p", + "id", + "4", + ], + ) + assert result.exit_code == 0 + assert result.output.strip().split("\n")[-1] == "Would affect 1 row" + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_output_column(test_db_and_path, drop): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = [ + { + "id": 1, + "dt": "5th October 2019 12:04", + "newcol": "5th Spooktober 2019 12:04", + }, + { + "id": 2, + "dt": "6th October 2019 00:05:06", + "newcol": "6th Spooktober 2019 00:05:06", + }, + {"id": 3, "dt": "", "newcol": ""}, + {"id": 4, "dt": None, "newcol": None}, + ] + if drop: + for row in expected: + del row["dt"] + assert list(db["example"].rows) == expected + + +@pytest.mark.parametrize( + "output_type,expected", + ( + ("text", [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ("float", [(1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0)]), + ("integer", [(1, 1), (2, 2), (3, 3), (4, 4)]), + (None, [(1, "1"), (2, "2"), (3, "3"), (4, "4")]), + ), +) +def test_convert_output_column_output_type(test_db_and_path, output_type, expected): + db, db_path = test_db_and_path + args = [ + "convert", + db_path, + "example", + "id", + "value", + "--output", + "new_id", + ] + if output_type: + args += ["--output-type", output_type] + result = CliRunner().invoke( + cli.cli, + args, + ) + assert 0 == result.exit_code, result.output + assert expected == list(db.execute("select id, new_id from example")) + + +@pytest.mark.parametrize( + "options,expected_error", + [ + ( + [ + "dt", + "id", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + ], + "Cannot use --output with more than one column", + ), + ( + [ + "dt", + "value.replace('October', 'Spooktober')", + "--output", + "newcol", + "--output-type", + "invalid", + ], + "Error: Invalid value for '--output-type'", + ), + ( + [ + "value.replace('October', 'Spooktober')", + ], + "Missing argument 'COLUMNS...'", + ), + ], +) +def test_convert_output_error(test_db_and_path, options, expected_error): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + ] + + options, + ) + assert result.exit_code != 0 + assert expected_error in result.output + + +@pytest.mark.parametrize("drop", (True, False)) +def test_convert_multi(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["creatures"].insert_all( + [ + {"id": 1, "name": "Simon"}, + {"id": 2, "name": "Cleo"}, + ], + pk="id", + ) + args = [ + "convert", + db_path, + "creatures", + "name", + "--multi", + '{"upper": value.upper(), "lower": value.lower()}', + ] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0, result.output + expected = [ + {"id": 1, "name": "Simon", "upper": "SIMON", "lower": "simon"}, + {"id": 2, "name": "Cleo", "upper": "CLEO", "lower": "cleo"}, + ] + if drop: + for row in expected: + del row["name"] + assert list(db["creatures"].rows) == expected + + +def test_convert_multi_complex_column_types(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["rows"].insert_all( + [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"id": 4}, + ], + pk="id", + ) + code = textwrap.dedent( + """ + if value == 1: + return {"is_str": "", "is_float": 1.2, "is_int": None} + elif value == 2: + return {"is_float": 1, "is_int": 12} + elif value == 3: + return {"is_bytes": b"blah"} + """ + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "rows", + "id", + "--multi", + code, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["rows"].rows) == [ + {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None}, + {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None}, + { + "id": 3, + "is_str": None, + "is_float": None, + "is_int": None, + "is_bytes": b"blah", + }, + {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, + ] + assert db["rows"].schema == ( + "CREATE TABLE [rows] (\n" + " [id] INTEGER PRIMARY KEY\n" + ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)" + ) + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_recipe_jsonsplit(tmpdir, delimiter): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = sqlite_utils.Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if delimiter: + code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) + args = ["convert", db_path, "example", "tags", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert list(db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected_array", + ( + (None, ["1", "2", "3"]), + ("float", [1.0, 2.0, 3.0]), + ("int", [1, 2, 3]), + ), +) +def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + if type: + code = "recipes.jsonsplit(value, type={})".format(type) + args = ["convert", db_path, "example", "records", code] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + assert json.loads(db["example"].get(1)["records"]) == expected_array + + +@pytest.mark.parametrize("drop", (True, False)) +def test_recipe_jsonsplit_output(fresh_db_and_path, drop): + db, db_path = fresh_db_and_path + db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + code = "r.jsonsplit(value)" + args = ["convert", db_path, "example", "records", code, "--output", "tags"] + if drop: + args += ["--drop"] + result = CliRunner().invoke(cli.cli, args) + assert 0 == result.exit_code, result.output + expected = { + "id": 1, + "records": "1,2,3", + "tags": '["1", "2", "3"]', + } + if drop: + del expected["records"] + assert db["example"].get(1) == expected + + +def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path): + args = ["convert", fresh_db_and_path[1], "example", "records", "value", "--drop"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: --drop can only be used with --output or --multi" in result.output + + +def test_cannot_use_multi_with_more_than_one_column(fresh_db_and_path): + args = [ + "convert", + fresh_db_and_path[1], + "example", + "records", + "othercol", + "value", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "Error: Cannot use --multi with more than one column" in result.output + + +def test_multi_with_bad_function(test_db_and_path): + args = [ + "convert", + test_db_and_path[1], + "example", + "dt", + "value.upper()", + "--multi", + ] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 1, result.output + assert "When using --multi code must return a Python dictionary" in result.output + + +def test_convert_where(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "str(value).upper()", + "--where", + "id = :id", + "-p", + "id", + 2, + ], + ) + assert result.exit_code == 0, result.output + assert list(db["example"].rows) == [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_convert_where_multi(fresh_db_and_path): + db, db_path = fresh_db_and_path + db["names"].insert_all( + [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "names", + "name", + '{"upper": value.upper()}', + "--where", + "id = :id", + "-p", + "id", + 2, + "--multi", + ], + ) + assert 0 == result.exit_code, result.output + assert list(db["names"].rows) == [ + {"id": 1, "name": "Cleo", "upper": None}, + {"id": 2, "name": "Bants", "upper": "BANTS"}, + ] diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py new file mode 100644 index 0000000..e465927 --- /dev/null +++ b/tests/test_cli_memory.py @@ -0,0 +1,222 @@ +import json + +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + + +def test_memory_basic(): + result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"1 + 1": 2}]' + + +@pytest.mark.parametrize("sql_from", ("test", "t", "t1")) +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv(tmpdir, sql_from, use_stdin): + content = "id,name\n1,Cleo\n2,Bants" + input = None + if use_stdin: + input = content + csv_path = "-" + if sql_from == "test": + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + open(csv_path, "w").write(content) + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + assert result.exit_code == 0 + assert ( + result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}' + ) + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_tsv(tmpdir, use_stdin): + data = "id\tname\n1\tCleo\n2\tBants" + if use_stdin: + input = data + path = "stdin:tsv" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.tsv") + open(path, "w").write(data) + path = path + ":tsv" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=input, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json(tmpdir, use_stdin): + data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]' + if use_stdin: + input = data + path = "stdin:json" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":json" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=input, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"name": "Bants", "age": None, "nested": None}, + {"name": "Dori", "age": 1, "nested": '{"nest": 1}'}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_json_nl(tmpdir, use_stdin): + data = '{"name": "Bants"}\n\n{"name": "Dori"}' + if use_stdin: + input = data + path = "stdin:nl" + sql_from = "stdin" + else: + input = None + path = str(tmpdir / "chickens.json") + open(path, "w").write(data) + path = path + ":nl" + sql_from = "chickens" + result = CliRunner().invoke( + cli.cli, + ["memory", path, "select * from {}".format(sql_from)], + input=input, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"name": "Bants"}, + {"name": "Dori"}, + ] + + +@pytest.mark.parametrize("use_stdin", (True, False)) +def test_memory_csv_encoding(tmpdir, use_stdin): + latin1_csv = ( + b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" + ) + input = None + if use_stdin: + input = latin1_csv + csv_path = "-" + sql_from = "stdin" + else: + csv_path = str(tmpdir / "test.csv") + with open(csv_path, "wb") as fp: + fp.write(latin1_csv) + sql_from = "test" + # Without --encoding should error: + assert ( + CliRunner() + .invoke( + cli.cli, + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + input=input, + ) + .exit_code + == 1 + ) + # With --encoding should work: + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"], + input=latin1_csv, + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == { + "date": "2020-03-04", + "name": "São Paulo", + "latitude": -23.561, + "longitude": -46.645, + } + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_dump(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--dump"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + "BEGIN TRANSACTION;\n" + 'CREATE TABLE "stdin" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n" + "INSERT INTO \"stdin\" VALUES(2,'Bants');\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];\n" + "COMMIT;" + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_schema(extra_args): + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--schema"], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + assert result.output.strip() == ( + 'CREATE TABLE "stdin" (\n' + " [id] INTEGER,\n" + " [name] TEXT\n" + ");\n" + "CREATE VIEW t1 AS select * from [stdin];\n" + "CREATE VIEW t AS select * from [stdin];" + ) + + +@pytest.mark.parametrize("extra_args", ([], ["select 1"])) +def test_memory_save(tmpdir, extra_args): + save_to = str(tmpdir / "save.db") + result = CliRunner().invoke( + cli.cli, + ["memory", "-"] + extra_args + ["--save", save_to], + input="id,name\n1,Cleo\n2,Bants", + ) + assert result.exit_code == 0 + db = Database(save_to) + assert list(db["stdin"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Bants"}, + ] + + +@pytest.mark.parametrize("option", ("-n", "--no-detect-types")) +def test_memory_no_detect_types(option): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin"] + [option], + input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + {"id": "1", "name": "Cleo", "weight": "45.5"}, + {"id": "2", "name": "Bants", "weight": "3.5"}, + ] diff --git a/tests/test_constructor.py b/tests/test_constructor.py index b3cd963..924df66 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,5 +1,4 @@ from sqlite_utils import Database -import pytest def test_recursive_triggers(): diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ebe2a50..d70f5c8 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -1,6 +1,3 @@ -import pytest - - def test_insert_conversion(fresh_db): table = fresh_db["table"] table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) diff --git a/tests/test_convert.py b/tests/test_convert.py new file mode 100644 index 0000000..796a08f --- /dev/null +++ b/tests/test_convert.py @@ -0,0 +1,117 @@ +from sqlite_utils.db import BadMultiValues +import pytest + + +@pytest.mark.parametrize( + "columns,fn,expected", + ( + ( + "title", + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "Abstract"}, + ), + ( + ["title", "abstract"], + lambda value: value.upper(), + {"title": "MIXED CASE", "abstract": "ABSTRACT"}, + ), + ), +) +def test_convert(fresh_db, columns, fn, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case", "abstract": "Abstract"}) + table.convert(columns, fn) + assert list(table.rows) == [expected] + + +@pytest.mark.parametrize( + "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) +) +def test_convert_where(fresh_db, where, where_args): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", lambda value: value.upper(), where=where, where_args=where_args + ) + assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] + + +@pytest.mark.parametrize( + "drop,expected", + ( + (False, {"title": "Mixed Case", "other": "MIXED CASE"}), + (True, {"other": "MIXED CASE"}), + ), +) +def test_convert_output(fresh_db, drop, expected): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert("title", lambda v: v.upper(), output="other", drop=drop) + assert list(table.rows) == [expected] + + +def test_convert_output_multiple_column_error(fresh_db): + table = fresh_db["table"] + with pytest.raises(AssertionError) as excinfo: + table.convert(["title", "other"], lambda v: v, output="out") + assert "output= can only be used with a single column" in str(excinfo.value) + + +@pytest.mark.parametrize( + "type,expected", + ( + (int, {"other": 123}), + (float, {"other": 123.0}), + ), +) +def test_convert_output_type(fresh_db, type, expected): + table = fresh_db["table"] + table.insert({"number": "123"}) + table.convert("number", lambda v: v, output="other", output_type=type, drop=True) + assert list(table.rows) == [expected] + + +def test_convert_multi(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + table.convert( + "title", lambda v: {"upper": v.upper(), "lower": v.lower()}, multi=True + ) + assert list(table.rows) == [ + {"title": "Mixed Case", "upper": "MIXED CASE", "lower": "mixed case"} + ] + + +def test_convert_multi_where(fresh_db): + table = fresh_db["table"] + table.insert_all( + [ + {"id": 1, "title": "One"}, + {"id": 2, "title": "Two"}, + ], + pk="id", + ) + table.convert( + "title", + lambda v: {"upper": v.upper(), "lower": v.lower()}, + multi=True, + where="id > ?", + where_args=[1], + ) + assert list(table.rows) == [ + {"id": 1, "lower": None, "title": "One", "upper": None}, + {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"}, + ] + + +def test_convert_multi_exception(fresh_db): + table = fresh_db["table"] + table.insert({"title": "Mixed Case"}) + with pytest.raises(BadMultiValues): + table.convert("title", lambda v: v.upper(), multi=True) diff --git a/tests/test_create.py b/tests/test_create.py index c1daf6b..ff36f90 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,7 +1,7 @@ from sqlite_utils.db import ( Index, Database, - ForeignKey, + DescIndex, AlterError, NoObviousTable, ForeignKey, @@ -147,8 +147,8 @@ def test_create_table_with_not_null(fresh_db): ) def test_create_table_from_example(fresh_db, example, expected_columns): people_table = fresh_db["people"] - assert None == people_table.last_rowid - assert None == people_table.last_pk + assert people_table.last_rowid is None + assert people_table.last_pk is None people_table.insert(example) assert 1 == people_table.last_rowid assert 1 == people_table.last_pk @@ -514,7 +514,7 @@ def test_insert_row_alter_table( def test_insert_row_alter_table_invalid_column_characters(fresh_db): table = fresh_db["table"] - rowid = table.insert({"foo": "bar"}).last_pk + table.insert({"foo": "bar"}).last_pk with pytest.raises(AssertionError): table.insert({"foo": "baz", "new_col[abc]": 1.2}, alter=True) @@ -739,6 +739,19 @@ def test_create_index_if_not_exists(fresh_db): dogs.create_index(["name"], if_not_exists=True) +def test_create_index_desc(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) + assert [] == dogs.indexes + dogs.create_index([DescIndex("age"), "name"]) + sql = fresh_db.execute( + "select sql from sqlite_master where name='idx_dogs_age_name'" + ).fetchone()[0] + assert sql == ( + "CREATE INDEX [idx_dogs_age_name]\n" " ON [dogs] ([age] desc, [name])" + ) + + @pytest.mark.parametrize( "data_structure", ( @@ -748,7 +761,7 @@ def test_create_index_if_not_exists(fresh_db): {"dictionary": {"nested": "complex"}}, collections.OrderedDict( [ - ("key1", {"nested": "complex"}), + ("key1", {"nested": ["cømplex"]}), ("key2", "foo"), ] ), @@ -762,6 +775,14 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): assert data_structure == json.loads(row[1]) +def test_insert_list_nested_unicode(fresh_db): + fresh_db["test"].insert( + {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id" + ) + row = fresh_db.execute("select id, data from test").fetchone() + assert row[1] == '{"key1": {"nested": ["cømplex"]}}' + + def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) @@ -805,8 +826,8 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) - rows = fresh_db.execute_returning_dicts("select * from test where i = 101") - assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows + rows = list(fresh_db.query("select * from test where i = 101")) + assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] def test_insert_ignore(fresh_db): @@ -817,8 +838,8 @@ def test_insert_ignore(fresh_db): # Using ignore=True should cause our insert to be silently ignored fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) # Only one row, and it should be bar=2, not bar=3 - rows = fresh_db.execute_returning_dicts("select * from test") - assert [{"id": 1, "bar": 2}] == rows + rows = list(fresh_db.query("select * from test")) + assert rows == [{"id": 1, "bar": 2}] def test_insert_hash_id(fresh_db): @@ -848,8 +869,6 @@ def test_works_with_pathlib_path(tmpdir): @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): - import numpy as np - df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) fresh_db["pandas"].insert_all(df.to_dict(orient="records")) assert [ @@ -969,6 +988,13 @@ def test_insert_all_empty_list(fresh_db): assert 1 == fresh_db["t"].count +def test_insert_all_single_column(fresh_db): + table = fresh_db["table"] + table.insert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) diff --git a/tests/test_docs.py b/tests/test_docs.py index 87b685e..d760629 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,10 +1,12 @@ -from sqlite_utils import cli +from click.testing import CliRunner +from sqlite_utils import cli, recipes from pathlib import Path import pytest import re docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+) ") +recipes_re = re.compile(r"r\.(\w+)\(") @pytest.fixture(scope="session") @@ -17,11 +19,36 @@ def documented_commands(): } +@pytest.fixture(scope="session") +def documented_recipes(): + rst = (docs_path / "cli.rst").read_text() + return set(recipes_re.findall(rst)) + + @pytest.mark.parametrize("command", cli.cli.commands.keys()) def test_commands_are_documented(documented_commands, command): assert command in documented_commands @pytest.mark.parametrize("command", cli.cli.commands.values()) -def test_commands_have_docstrings(command): - assert command.__doc__, "{} is missing a docstring".format(command) +def test_commands_have_help(command): + assert command.help, "{} is missing its help".format(command) + + +def test_convert_help(): + result = CliRunner().invoke(cli.cli, ["convert", "--help"]) + assert result.exit_code == 0 + for expected in ( + "r.jsonsplit(value, ", + "r.parsedate(value, ", + "r.parsedatetime(value, ", + ): + assert expected in result.output + + +@pytest.mark.parametrize( + "recipe", + [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], +) +def test_recipes_are_documented(documented_recipes, recipe): + assert recipe in documented_recipes diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index b70378e..d724e80 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -14,8 +14,31 @@ def test_enable_counts_specific_table(fresh_db): # Now enable counts foo.enable_counts() assert foo.triggers_dict == { - "foo_counts_insert": "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) + 1\n );\nEND", - "foo_counts_delete": "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\nBEGIN\n INSERT OR REPLACE INTO [_counts]\n VALUES (\n 'foo',\n COALESCE(\n (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n 0\n ) - 1\n );\nEND", + "foo_counts_insert": ( + "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) + 1\n" + " );\n" + "END" + ), + "foo_counts_delete": ( + "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n" + "BEGIN\n" + " INSERT OR REPLACE INTO [_counts]\n" + " VALUES (\n" + " 'foo',\n" + " COALESCE(\n" + " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" + " 0\n" + " ) - 1\n" + " );\n" + "END" + ), } assert fresh_db.table_names() == ["foo", "_counts"] assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] @@ -109,7 +132,7 @@ def test_uses_counts_after_enable_counts(counts_db_path): assert db["foo"].count == 1 assert logged == [ ("select name from sqlite_master where type = 'view'", None), - ("select count(*) from [foo]", None), + ("select count(*) from [foo]", []), ] logged.clear() assert not db.use_counts_table diff --git a/tests/test_extract.py b/tests/test_extract.py index 9eae704..10b5b09 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, InvalidColumns +from sqlite_utils.db import InvalidColumns import itertools import pytest @@ -126,12 +126,25 @@ def test_extract_rowid_table(fresh_db): fresh_db["tree"].extract(["common_name", "latin_name"]) assert fresh_db["tree"].schema == ( 'CREATE TABLE "tree" (\n' - " [rowid] INTEGER PRIMARY KEY,\n" " [name] TEXT,\n" " [common_name_latin_name_id] INTEGER,\n" " FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n" ")" ) + assert ( + fresh_db.execute( + """ + select + tree.name, + common_name_latin_name.common_name, + common_name_latin_name.latin_name + from tree + join common_name_latin_name + on tree.common_name_latin_name_id = common_name_latin_name.id + """ + ).fetchall() + == [("Tree 1", "Palm", "Arecaceae")] + ) def test_reuse_lookup_table(fresh_db): diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 0edd002..cca16ba 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, ForeignKey +from sqlite_utils.db import Index import pytest diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index 5759c5e..f12f865 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -2,6 +2,7 @@ from hypothesis import given import hypothesis.strategies as st import sqlite_utils + # SQLite integers are -(2^63) to 2^63 - 1 @given(st.integers(-9223372036854775808, 9223372036854775807)) def test_roundtrip_integers(integer): diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1a11d2c..1e30a8d 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -2,9 +2,11 @@ from sqlite_utils import cli, Database from click.testing import CliRunner import os import pathlib +import pytest -def test_insert_files(): +@pytest.mark.parametrize("silent", (False, True)) +def test_insert_files(silent): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") @@ -34,7 +36,10 @@ def test_insert_files(): cols += ["-c", "{}:{}".format(coltype, coltype)] result = runner.invoke( cli.cli, - ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"], + ["insert-files", db_path, "files", str(tmpdir)] + + cols + + ["--pk", "path"] + + (["--silent"] if silent else []), catch_exceptions=False, ) assert result.exit_code == 0, result.stdout diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 73102f8..dce8afc 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import Index, View, Database +from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest @@ -33,7 +33,7 @@ def test_detect_fts(existing_db): assert "woo_fts" == existing_db["woo_fts"].detect_fts() assert "woo2_fts" == existing_db["woo2"].detect_fts() assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() - assert None == existing_db["foo"].detect_fts() + assert existing_db["foo"].detect_fts() is None def test_tables(existing_db): @@ -52,7 +52,14 @@ def test_views(fresh_db): def test_count(existing_db): - assert 3 == existing_db["foo"].count + assert existing_db["foo"].count == 3 + assert existing_db["foo"].count_where() == 3 + assert existing_db["foo"].execute_count() == 3 + + +def test_count_where(existing_db): + assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 + assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 def test_columns(existing_db): @@ -62,8 +69,12 @@ def test_columns(existing_db): ] -def test_schema(existing_db): - assert "CREATE TABLE foo (text TEXT)" == existing_db["foo"].schema +def test_table_schema(existing_db): + assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" + + +def test_database_schema(existing_db): + assert existing_db.schema == "CREATE TABLE foo (text TEXT);" def test_table_repr(fresh_db): @@ -93,6 +104,33 @@ def test_indexes(fresh_db): ] == fresh_db["Gosh"].indexes +def test_xindexes(fresh_db): + fresh_db.executescript( + """ + create table Gosh (c1 text, c2 text, c3 text); + create index Gosh_c1 on Gosh(c1); + create index Gosh_c2c3 on Gosh(c2, c3 desc); + """ + ) + assert fresh_db["Gosh"].xindexes == [ + XIndex( + name="Gosh_c2c3", + columns=[ + XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1), + XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + XIndex( + name="Gosh_c1", + columns=[ + XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ), + ] + + @pytest.mark.parametrize( "column,expected_table_guess", ( @@ -144,9 +182,21 @@ def test_triggers_and_triggers_dict(fresh_db): (t.name, t.table) for t in fresh_db["authors"].triggers } expected_triggers = { - "authors_ai": "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", - "authors_ad": "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\nEND", - "authors_au": "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND", + "authors_ai": ( + "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n" + "END" + ), + "authors_ad": ( + "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + "END" + ), + "authors_au": ( + "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n" + " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" + " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND" + ), } assert authors.triggers_dict == expected_triggers assert fresh_db["other"].triggers == [] @@ -206,3 +256,11 @@ def test_virtual_table_using(sql, expected_name, expected_using): db = Database(memory=True) db.execute(sql) assert db[expected_name].virtual_table_using == expected_using + + +def test_use_rowid(): + db = Database(memory=True) + db["rowid_table"].insert({"name": "Cleo"}) + db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") + assert db["rowid_table"].use_rowid + assert not db["regular_table"].use_rowid diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..fe79cc0 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,17 @@ +import types + + +def test_query(fresh_db): + fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + results = fresh_db.query("select * from dogs order by name desc") + assert isinstance(results, types.GeneratorType) + assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] + + +def test_execute_returning_dicts(fresh_db): + # Like db.query() but returns a list, included for backwards compatibility + # see https://github.com/simonw/sqlite-utils/issues/290 + fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + assert fresh_db.execute_returning_dicts("select * from test") == [ + {"id": 1, "bar": 2} + ] diff --git a/tests/test_recipes.py b/tests/test_recipes.py new file mode 100644 index 0000000..89240a2 --- /dev/null +++ b/tests/test_recipes.py @@ -0,0 +1,108 @@ +from sqlite_utils import recipes +import json +import pytest + + +@pytest.fixture +def dates_db(fresh_db): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "5th October 2019 12:04"}, + {"id": 2, "dt": "6th October 2019 00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ], + pk="id", + ) + return fresh_db + + +def test_parsedate(dates_db): + dates_db["example"].convert("dt", recipes.parsedate) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05"}, + {"id": 2, "dt": "2019-10-06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +def test_parsedatetime(dates_db): + dates_db["example"].convert("dt", recipes.parsedatetime) + assert list(dates_db["example"].rows) == [ + {"id": 1, "dt": "2019-10-05T12:04:00"}, + {"id": 2, "dt": "2019-10-06T00:05:06"}, + {"id": 3, "dt": ""}, + {"id": 4, "dt": None}, + ] + + +@pytest.mark.parametrize( + "recipe,kwargs,expected", + ( + ("parsedate", {}, "2005-03-04"), + ("parsedate", {"dayfirst": True}, "2005-04-03"), + ("parsedatetime", {}, "2005-03-04T00:00:00"), + ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"), + ), +) +def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "03/04/05"}, + ], + pk="id", + ) + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) + ) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "dt": expected}, + ] + + +@pytest.mark.parametrize("delimiter", [None, ";", "-"]) +def test_jsonsplit(fresh_db, delimiter): + fresh_db["example"].insert_all( + [ + {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, + {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if delimiter is not None: + + def fn(value): + return recipes.jsonsplit(value, delimiter=delimiter) + + fresh_db["example"].convert("tags", fn) + assert list(fresh_db["example"].rows) == [ + {"id": 1, "tags": '["foo", "bar"]'}, + {"id": 2, "tags": '["bar", "baz"]'}, + ] + + +@pytest.mark.parametrize( + "type,expected", + ( + (None, ["1", "2", "3"]), + (float, [1.0, 2.0, 3.0]), + (int, [1, 2, 3]), + ), +) +def test_jsonsplit_type(fresh_db, type, expected): + fresh_db["example"].insert_all( + [ + {"id": 1, "records": "1,2,3"}, + ], + pk="id", + ) + fn = recipes.jsonsplit + if type is not None: + + def fn(value): + return recipes.jsonsplit(value, type=type) + + fresh_db["example"].convert("records", fn) + assert json.loads(fresh_db["example"].get(1)["records"]) == expected diff --git a/tests/test_recreate.py b/tests/test_recreate.py index ddef115..504a0b8 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory(): def test_recreate_not_allowed_for_connection(): conn = sqlite3.connect(":memory:") with pytest.raises(AssertionError): - db = Database(conn, recreate=True) + Database(conn, recreate=True) @pytest.mark.parametrize( diff --git a/tests/test_register_function.py b/tests/test_register_function.py index e6d977a..19af0b6 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -1,3 +1,4 @@ +# flake8: noqa import pytest import sys from unittest.mock import MagicMock @@ -55,14 +56,14 @@ def test_register_function_replace(fresh_db): # This will fail to replace the function: @fresh_db.register_function() - def one(): + def one(): # noqa return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): + def one(): # noqa return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] diff --git a/tests/test_rows.py b/tests/test_rows.py index 73bd94f..a8a4ca0 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -1,4 +1,3 @@ -from sqlite_utils.db import Index, View import pytest @@ -13,6 +12,7 @@ def test_rows(existing_db): [ ("name = ?", ["Pancakes"], {2}), ("age > ?", [3], {1}), + ("age > :age", {"age": 3}, {1}), ("name is not null", [], {1, 2}), ("is_good = ?", [True], {1, 2}), ], diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 094551a..d3ff22d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,4 +1,3 @@ -import pytest from sqlite_utils import Database @@ -32,7 +31,9 @@ def test_tracer(): def test_with_tracer(): collected = [] - tracer = lambda sql, params: collected.append((sql, params)) + + def tracer(sql, params): + return collected.append((sql, params)) db = Database(memory=True) @@ -48,13 +49,39 @@ def test_with_tracer(): assert collected == [ ("select name from sqlite_master where type = 'view'", None), ( - "SELECT name FROM sqlite_master\n WHERE rootpage = 0\n AND (\n sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n OR (\n tbl_name = \"dogs\"\n AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n )\n )", + ( + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE '%VIRTUAL TABLE%USING FTS%content=%dogs%'\n" + " OR (\n" + ' tbl_name = "dogs"\n' + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )" + ), None, ), ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - "with original as (\n select\n rowid,\n *\n from [dogs]\n)\nselect\n [original].*\nfrom\n [original]\n join [dogs_fts] on [original].rowid = [dogs_fts].rowid\nwhere\n [dogs_fts] match :query\norder by\n [dogs_fts].rank", + ( + "with original as (\n" + " select\n" + " rowid,\n" + " *\n" + " from [dogs]\n" + ")\n" + "select\n" + " [original].*\n" + "from\n" + " [original]\n" + " join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n" + "where\n" + " [dogs_fts] match :query\n" + "order by\n" + " [dogs_fts].rank" + ), {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index b3ef009..06e5729 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -89,9 +89,14 @@ import pytest ], ) @pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) -def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): +def test_transform_sql_table_with_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): captured = [] - tracer = lambda sql, params: captured.append((sql, params)) + + def tracer(sql, params): + return captured.append((sql, params)) + dogs = fresh_db["dogs"] if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") @@ -111,7 +116,80 @@ def test_transform_sql(fresh_db, params, expected_sql, use_pragma_foreign_keys): assert ("PRAGMA foreign_keys=1;", None) not in captured -def test_transform_sql_rowid_to_id(fresh_db): +@pytest.mark.parametrize( + "params,expected_sql", + [ + # Identity transform - nothing changes + ( + {}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Change column type + ( + {"types": {"age": int}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [age] INTEGER\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Rename a column + ( + {"rename": {"age": "dog_age"}}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER,\n [name] TEXT,\n [dog_age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [dog_age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + # Make ID a primary key + ( + {"pk": "id"}, + [ + "CREATE TABLE [dogs_new_suffix] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] TEXT\n);", + "INSERT INTO [dogs_new_suffix] ([id], [name], [age])\n SELECT [id], [name], [age] FROM [dogs];", + "DROP TABLE [dogs];", + "ALTER TABLE [dogs_new_suffix] RENAME TO [dogs];", + ], + ), + ], +) +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_sql_table_with_no_primary_key( + fresh_db, params, expected_sql, use_pragma_foreign_keys +): + captured = [] + + def tracer(sql, params): + return captured.append((sql, params)) + + dogs = fresh_db["dogs"] + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + assert sql == expected_sql + # Check that .transform() runs without exceptions: + with fresh_db.tracer(tracer): + dogs.transform(**params) + # If use_pragma_foreign_keys, check that we did the right thing + if use_pragma_foreign_keys: + assert ("PRAGMA foreign_keys=0;", None) in captured + assert captured[-2] == ("PRAGMA foreign_key_check;", None) + assert captured[-1] == ("PRAGMA foreign_keys=1;", None) + else: + assert ("PRAGMA foreign_keys=0;", None) not in captured + assert ("PRAGMA foreign_keys=1;", None) not in captured + + +def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 100c6c1..09bdacc 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -21,6 +21,13 @@ def test_upsert_all(fresh_db): assert table.last_pk is None +def test_upsert_all_single_column(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"name": "Cleo"}], pk="name") + assert [{"name": "Cleo"}] == list(table.rows) + assert table.pks == ["name"] + + def test_upsert_error_if_no_pk(fresh_db): table = fresh_db["table"] with pytest.raises(PrimaryKeyRequired): @@ -47,7 +54,7 @@ def test_upsert_compound_primary_key(fresh_db): ], pk=("species", "id"), ) - assert None == table.last_pk + assert table.last_pk is None table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id")) assert ("dog", 1) == table.last_pk assert [ diff --git a/tests/test_wal.py b/tests/test_wal.py index 1303eed..23ca144 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,6 +1,5 @@ import pytest from sqlite_utils import Database -import sqlite3 @pytest.fixture