Release 3.27

Refs #434, #435, #436, #440, #441, #442, #443

docs/changelog.rst

@@ -2,6 +2,19 @@
  Changelog
 ===========
 
+.. _v3_27:
+
+3.27 (2022-06-14)
+-----------------
+
+- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`435`)
+- Code examples in documentation now have a "copy to clipboard" button. (:issue:`436`)
+- ``sqlite_utils.utils.utils.rows_from_file()`` is now a documented API, see :ref:`python_api_rows_from_file`. (:issue:`443`)
+- ``rows_from_file()`` has two new parameters to help handle CSV files with rows that contain more values than are listed in that CSV file's headings: ``ignore_extrasTrue`` and ``extras_key="name-of-key"``. (:issue:`440`)
+- ``sqlite_utils.utils.maximize_csv_field_size_limit()`` helper function for increasing the field size limit for reading CSV files to its maximum, see :ref:`python_api_maximize_csv_field_size_limit`. (:issue:`442`)
+- ``table.search(where=, where_args=)`` parameters for adding additional ``WHERE`` clauses to a search query. The ``where=`` parameter is available on ``table.search_sql(...)`` as well. See :ref:`python_api_fts_search`. (:issue:`441`)
+- Fixed bug where ``table.detect_fts()`` and other search-related functions could fail if two FTS-enabled tables had names that were prefixes of each other. (:issue:`434`)
+
 .. _v3_26_1:
 
 3.26.1 (2022-05-02)

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import io
 import os
 
-VERSION = "3.26.1"
+VERSION = "3.27"
 
 
 def get_long_description():

sqlite_utils/cli.py

@@ -6,7 +6,7 @@ import hashlib
 import pathlib
 import sqlite_utils
 from sqlite_utils.db import AlterError, BadMultiValues, DescIndex
-from sqlite_utils.utils import maximize_csv_field_size_limit
+from sqlite_utils.utils import maximize_csv_field_size_limit, _extra_key_strategy_row
 from sqlite_utils import recipes
 import textwrap
 import inspect
@@ -797,6 +797,15 @@ _import_options = (
         "--encoding",
         help="Character encoding for input, defaults to utf-8",
     ),
+    click.option(
+        "--ignore-extras",
+        is_flag=True,
+        help="If a CSV line has more than the expected number of values, ignore the extras",
+    ),
+    click.option(
+        "--extras-key",
+        help="If a CSV line has more than the expected number of values put them in a list in this column",
+    ),
 )
 
 
@@ -885,6 +894,8 @@ def insert_upsert_implementation(
     sniff,
     no_headers,
     encoding,
+    ignore_extras,
+    extras_key,
     batch_size,
     alter,
     upsert,
@@ -909,6 +920,10 @@ def insert_upsert_implementation(
         raise click.ClickException("--flatten cannot be used with --csv or --tsv")
     if encoding and not (csv or tsv):
         raise click.ClickException("--encoding must be used with --csv or --tsv")
+    if ignore_extras and extras_key:
+        raise click.ClickException(
+            "--ignore-extras and --extras-key cannot be used together"
+        )
     if pk and len(pk) == 1:
         pk = pk[0]
     encoding = encoding or "utf-8-sig"
@@ -942,7 +957,10 @@ def insert_upsert_implementation(
                 reader = itertools.chain([first_row], reader)
             else:
                 headers = first_row
-            docs = (dict(zip(headers, row)) for row in reader)
+            docs = (
+                _extra_key_strategy_row(headers, row, ignore_extras, extras_key)
+                for row in reader
+            )
             if detect_types:
                 tracker = TypeTracker()
                 docs = tracker.wrap(docs)
@@ -1101,6 +1119,8 @@ def insert(
     sniff,
     no_headers,
     encoding,
+    ignore_extras,
+    extras_key,
     batch_size,
     alter,
     detect_types,
@@ -1176,6 +1196,8 @@ def insert(
             sniff,
             no_headers,
             encoding,
+            ignore_extras,
+            extras_key,
             batch_size,
             alter=alter,
             upsert=False,
@@ -1214,6 +1236,8 @@ def upsert(
     sniff,
     no_headers,
     encoding,
+    ignore_extras,
+    extras_key,
     alter,
     not_null,
     default,
@@ -1254,6 +1278,8 @@ def upsert(
             sniff,
             no_headers,
             encoding,
+            ignore_extras,
+            extras_key,
             batch_size,
             alter=alter,
             upsert=True,
@@ -1297,6 +1323,8 @@ def bulk(
     sniff,
     no_headers,
     encoding,
+    ignore_extras,
+    extras_key,
     load_extension,
 ):
     """
@@ -1331,6 +1359,8 @@ def bulk(
             sniff=sniff,
             no_headers=no_headers,
             encoding=encoding,
+            ignore_extras=ignore_extras,
+            extras_key=extras_key,
             batch_size=batch_size,
             alter=False,
             upsert=False,

sqlite_utils/utils.py

@@ -9,7 +9,7 @@ import json
 import os
 import sys
 from . import recipes
-from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type
+from typing import Any, Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type
 
 import click
 
@@ -219,6 +219,25 @@ def _extra_key_strategy(
             yield row
 
 
+def _extra_key_strategy_row(
+    headers: Iterable[str],
+    row: Iterable[Any],
+    ignore_extras: Optional[bool] = False,
+    extras_key: Optional[str] = None,
+) -> dict:
+    # Logic for handling CSV rows with more values than there are headings
+    if len(headers) <= len(row) or ignore_extras:
+        return dict(zip(headers, row))
+    # There are more row items than headers, what to do with the extras?
+    if not ignore_extras:
+        extras = row[len(headers) :]
+        raise RowError("Row {} contained these extra values: {}".format(row, extras))
+    if extras_key:
+        d = dict(zip(headers, row))
+        d[extras_key] = row[len(headers) :]
+        return d
+
+
 def rows_from_file(
     fp: BinaryIO,
     format: Optional[Format] = None,

tests/test_cli_insert.py

@@ -222,6 +222,42 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir):
     assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows)
 
 
+@pytest.mark.parametrize(
+    "options,expected_rows,expected_error",
+    [
+        (["--csv", "--ignore-extras"], [{"id": "1", "name": "Cleo"}], None),
+        (
+            ["--csv", "--extras-key", "x"],
+            [{"id": "1", "name": "Cleo", "x": '["extra"]'}],
+            None,
+        ),
+        (["--csv"], None, "Oh no an extra field found"),
+        (
+            ["--csv", "--extras-key", "x", "--ignore-extras"],
+            None,
+            "Error: --ignore-extras and --extras-key cannot be used together\n",
+        ),
+    ],
+)
+def test_insert_csv_with_extra_fields(
+    options, expected_rows, expected_error, db_path, tmpdir
+):
+    db = Database(db_path)
+    file_path = str(tmpdir / "insert.csv")
+    open(file_path, "w").write("id,name\n1,Cleo,extra")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", file_path] + options,
+        catch_exceptions=False,
+    )
+    if expected_error:
+        assert result.exit_code == 2
+        assert result.output == expected_error
+    else:
+        assert result.exit_code == 0
+        assert list(db["data"].rows) == expected_rows
+
+
 @pytest.mark.parametrize(
     "options",
     (