Compare commits

...

6 commits

Author SHA1 Message Date
Simon Willison
44edc65328 Bump to latest ty 2026-05-17 16:46:27 -07:00
Simon Willison
5b0c27036e Update tabulate and use that in cog 2026-05-17 16:33:45 -07:00
Simon Willison
37847e0c27 Bump Black version 2026-05-17 16:29:45 -07:00
Simon Willison
cb143c0bcf Ran latest Black 2026-05-17 16:27:07 -07:00
Simon Willison
b0b6adb288
Run ty check only on 3.14 2026-05-17 16:25:02 -07:00
Simon Willison
d5485459c6
Add Python 3.15-dev to test matrix 2026-05-17 16:17:36 -07:00
17 changed files with 175 additions and 216 deletions

View file

@ -10,7 +10,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
strategy: strategy:
matrix: matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15-dev"]
numpy: [0, 1] numpy: [0, 1]
os: [ubuntu-latest, macos-latest, windows-latest, macos-14] os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
steps: steps:
@ -46,7 +46,7 @@ jobs:
- name: run flake8 - name: run flake8
run: flake8 run: flake8
- name: run ty - name: run ty
if: matrix.os != 'windows-latest' if: matrix.os != 'windows-latest' && matrix.python-version == '3.14'
run: | run: |
pip install uv pip install uv
uv run ty check sqlite_utils uv run ty check sqlite_utils

View file

@ -117,15 +117,15 @@ See :ref:`cli_query`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid,
grid, heavy_grid, heavy_outline, html, jira, fancy_outline, github, grid, heavy_grid,
latex, latex_booktabs, latex_longtable, latex_raw, heavy_outline, html, jira, latex, latex_booktabs,
mediawiki, mixed_grid, mixed_outline, moinmoin, latex_longtable, latex_raw, mediawiki, mixed_grid,
orgtbl, outline, pipe, plain, presto, pretty, mixed_outline, moinmoin, orgtbl, outline, pipe,
psql, rounded_grid, rounded_outline, rst, simple, plain, presto, pretty, psql, rounded_grid,
simple_grid, simple_outline, textile, tsv, rounded_outline, rst, simple, simple_grid,
unsafehtml, youtrack simple_outline, textile, tsv, unsafehtml, youtrack
--json-cols Detect JSON cols and output them as JSON, not --json-cols Detect JSON cols and output them as JSON, not
escaped strings escaped strings
-r, --raw Raw output, first column of first row -r, --raw Raw output, first column of first row
@ -186,15 +186,15 @@ See :ref:`cli_memory`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid,
grid, heavy_grid, heavy_outline, html, jira, fancy_outline, github, grid, heavy_grid,
latex, latex_booktabs, latex_longtable, latex_raw, heavy_outline, html, jira, latex, latex_booktabs,
mediawiki, mixed_grid, mixed_outline, moinmoin, latex_longtable, latex_raw, mediawiki, mixed_grid,
orgtbl, outline, pipe, plain, presto, pretty, mixed_outline, moinmoin, orgtbl, outline, pipe,
psql, rounded_grid, rounded_outline, rst, simple, plain, presto, pretty, psql, rounded_grid,
simple_grid, simple_outline, textile, tsv, rounded_outline, rst, simple, simple_grid,
unsafehtml, youtrack simple_outline, textile, tsv, unsafehtml, youtrack
--json-cols Detect JSON cols and output them as JSON, not --json-cols Detect JSON cols and output them as JSON, not
escaped strings escaped strings
-r, --raw Raw output, first column of first row -r, --raw Raw output, first column of first row
@ -425,14 +425,15 @@ See :ref:`cli_search`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid, fancy_outline,
grid, heavy_grid, heavy_outline, html, jira, latex, github, grid, heavy_grid, heavy_outline, html, jira,
latex_booktabs, latex_longtable, latex_raw, mediawiki, latex, latex_booktabs, latex_longtable, latex_raw,
mixed_grid, mixed_outline, moinmoin, orgtbl, outline, mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl,
pipe, plain, presto, pretty, psql, rounded_grid, outline, pipe, plain, presto, pretty, psql,
rounded_outline, rst, simple, simple_grid, rounded_grid, rounded_outline, rst, simple,
simple_outline, textile, tsv, unsafehtml, youtrack simple_grid, simple_outline, textile, tsv, unsafehtml,
youtrack
--json-cols Detect JSON cols and output them as JSON, not escaped --json-cols Detect JSON cols and output them as JSON, not escaped
strings strings
--load-extension TEXT Path to SQLite extension, with optional :entrypoint --load-extension TEXT Path to SQLite extension, with optional :entrypoint
@ -690,14 +691,15 @@ See :ref:`cli_tables`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid, fancy_outline,
grid, heavy_grid, heavy_outline, html, jira, latex, github, grid, heavy_grid, heavy_outline, html, jira,
latex_booktabs, latex_longtable, latex_raw, mediawiki, latex, latex_booktabs, latex_longtable, latex_raw,
mixed_grid, mixed_outline, moinmoin, orgtbl, outline, mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl,
pipe, plain, presto, pretty, psql, rounded_grid, outline, pipe, plain, presto, pretty, psql,
rounded_outline, rst, simple, simple_grid, rounded_grid, rounded_outline, rst, simple,
simple_outline, textile, tsv, unsafehtml, youtrack simple_grid, simple_outline, textile, tsv, unsafehtml,
youtrack
--json-cols Detect JSON cols and output them as JSON, not escaped --json-cols Detect JSON cols and output them as JSON, not escaped
strings strings
--columns Include list of columns for each table --columns Include list of columns for each table
@ -731,14 +733,15 @@ See :ref:`cli_views`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid, fancy_outline,
grid, heavy_grid, heavy_outline, html, jira, latex, github, grid, heavy_grid, heavy_outline, html, jira,
latex_booktabs, latex_longtable, latex_raw, mediawiki, latex, latex_booktabs, latex_longtable, latex_raw,
mixed_grid, mixed_outline, moinmoin, orgtbl, outline, mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl,
pipe, plain, presto, pretty, psql, rounded_grid, outline, pipe, plain, presto, pretty, psql,
rounded_outline, rst, simple, simple_grid, rounded_grid, rounded_outline, rst, simple,
simple_outline, textile, tsv, unsafehtml, youtrack simple_grid, simple_outline, textile, tsv, unsafehtml,
youtrack
--json-cols Detect JSON cols and output them as JSON, not escaped --json-cols Detect JSON cols and output them as JSON, not escaped
strings strings
--columns Include list of columns for each view --columns Include list of columns for each view
@ -777,15 +780,15 @@ See :ref:`cli_rows`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid,
grid, heavy_grid, heavy_outline, html, jira, fancy_outline, github, grid, heavy_grid,
latex, latex_booktabs, latex_longtable, latex_raw, heavy_outline, html, jira, latex, latex_booktabs,
mediawiki, mixed_grid, mixed_outline, moinmoin, latex_longtable, latex_raw, mediawiki, mixed_grid,
orgtbl, outline, pipe, plain, presto, pretty, mixed_outline, moinmoin, orgtbl, outline, pipe,
psql, rounded_grid, rounded_outline, rst, simple, plain, presto, pretty, psql, rounded_grid,
simple_grid, simple_outline, textile, tsv, rounded_outline, rst, simple, simple_grid,
unsafehtml, youtrack simple_outline, textile, tsv, unsafehtml, youtrack
--json-cols Detect JSON cols and output them as JSON, not --json-cols Detect JSON cols and output them as JSON, not
escaped strings escaped strings
--load-extension TEXT Path to SQLite extension, with optional --load-extension TEXT Path to SQLite extension, with optional
@ -817,14 +820,15 @@ See :ref:`cli_triggers`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid, fancy_outline,
grid, heavy_grid, heavy_outline, html, jira, latex, github, grid, heavy_grid, heavy_outline, html, jira,
latex_booktabs, latex_longtable, latex_raw, mediawiki, latex, latex_booktabs, latex_longtable, latex_raw,
mixed_grid, mixed_outline, moinmoin, orgtbl, outline, mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl,
pipe, plain, presto, pretty, psql, rounded_grid, outline, pipe, plain, presto, pretty, psql,
rounded_outline, rst, simple, simple_grid, rounded_grid, rounded_outline, rst, simple,
simple_outline, textile, tsv, unsafehtml, youtrack simple_grid, simple_outline, textile, tsv, unsafehtml,
youtrack
--json-cols Detect JSON cols and output them as JSON, not escaped --json-cols Detect JSON cols and output them as JSON, not escaped
strings strings
--load-extension TEXT Path to SQLite extension, with optional :entrypoint --load-extension TEXT Path to SQLite extension, with optional :entrypoint
@ -856,14 +860,15 @@ See :ref:`cli_indexes`.
--tsv Output TSV --tsv Output TSV
--no-headers Omit CSV headers --no-headers Omit CSV headers
-t, --table Output as a formatted table -t, --table Output as a formatted table
--fmt TEXT Table format - one of asciidoc, double_grid, --fmt TEXT Table format - one of asciidoc, colon_grid,
double_outline, fancy_grid, fancy_outline, github, double_grid, double_outline, fancy_grid, fancy_outline,
grid, heavy_grid, heavy_outline, html, jira, latex, github, grid, heavy_grid, heavy_outline, html, jira,
latex_booktabs, latex_longtable, latex_raw, mediawiki, latex, latex_booktabs, latex_longtable, latex_raw,
mixed_grid, mixed_outline, moinmoin, orgtbl, outline, mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl,
pipe, plain, presto, pretty, psql, rounded_grid, outline, pipe, plain, presto, pretty, psql,
rounded_outline, rst, simple, simple_grid, rounded_grid, rounded_outline, rst, simple,
simple_outline, textile, tsv, unsafehtml, youtrack simple_grid, simple_outline, textile, tsv, unsafehtml,
youtrack
--json-cols Detect JSON cols and output them as JSON, not escaped --json-cols Detect JSON cols and output them as JSON, not escaped
strings strings
--load-extension TEXT Path to SQLite extension, with optional :entrypoint --load-extension TEXT Path to SQLite extension, with optional :entrypoint

View file

@ -257,6 +257,7 @@ Available ``--fmt`` options are:
.. ]]] .. ]]]
- ``asciidoc`` - ``asciidoc``
- ``colon_grid``
- ``double_grid`` - ``double_grid``
- ``double_outline`` - ``double_outline``
- ``fancy_grid`` - ``fancy_grid``

View file

@ -33,7 +33,7 @@ dependencies = [
[dependency-groups] [dependency-groups]
dev = [ dev = [
"black>=24.1.1", "black>=26.3.1",
"cogapp", "cogapp",
"hypothesis", "hypothesis",
"pytest", "pytest",
@ -47,7 +47,9 @@ dev = [
# flake8 # flake8
"flake8", "flake8",
"flake8-pyproject", "flake8-pyproject",
"ty", "ty>=0.0.37",
# For stable cog:
"tabulate>=0.10.0",
] ]
docs = [ docs = [
"beanbag-docutils>=2.0", "beanbag-docutils>=2.0",

View file

@ -1,7 +1,7 @@
import base64 import base64
from typing import Any from typing import Any
import click import click
from click_default_group import DefaultGroup # type: ignore from click_default_group import DefaultGroup
from datetime import datetime, timezone from datetime import datetime, timezone
import hashlib import hashlib
import pathlib import pathlib
@ -42,7 +42,6 @@ from .utils import (
TypeTracker, TypeTracker,
) )
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
@ -223,7 +222,7 @@ def tables(
else: else:
items = db.table_names(fts4=fts4, fts5=fts5) items = db.table_names(fts4=fts4, fts5=fts5)
for name in items: for name in items:
row = [name] row: list[Any] = [name]
if counts: if counts:
row.append(method(name).count) row.append(method(name).count)
if columns: if columns:
@ -2122,8 +2121,9 @@ def _execute_query(
cursor = [[cursor.rowcount]] cursor = [[cursor.rowcount]]
else: else:
headers = [c[0] for c in cursor.description] headers = [c[0] for c in cursor.description]
cursor_or_rows: Any = cursor
if raw: if raw:
row = cursor.fetchone() # type: ignore[union-attr] row = cursor_or_rows.fetchone()
data = row[0] if row else None data = row[0] if row else None
if isinstance(data, bytes): if isinstance(data, bytes):
sys.stdout.buffer.write(data) sys.stdout.buffer.write(data)
@ -2911,8 +2911,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least
) )
details = ( details = (
( (
textwrap.dedent( textwrap.dedent("""
"""
{table}.{column}: ({i}/{total}) {table}.{column}: ({i}/{total})
Total rows: {total_rows} Total rows: {total_rows}
@ -2920,8 +2919,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least
Blank rows: {num_blank} Blank rows: {num_blank}
Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered}
""" """)
)
.strip() .strip()
.format( .format(
i=i + 1, i=i + 1,
@ -2968,8 +2966,7 @@ def uninstall(packages, yes):
def _generate_convert_help(): def _generate_convert_help():
help = textwrap.dedent( help = textwrap.dedent("""
"""
Convert columns using Python code you supply. For example: Convert columns using Python code you supply. For example:
\b \b
@ -2982,8 +2979,7 @@ def _generate_convert_help():
Use "-" for CODE to read Python code from standard input. Use "-" for CODE to read Python code from standard input.
The following common operations are available as recipe functions: The following common operations are available as recipe functions:
""" """).strip()
).strip()
recipe_names = [ recipe_names = [
n n
for n in dir(recipes) for n in dir(recipes)
@ -2997,15 +2993,13 @@ def _generate_convert_help():
name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip())
) )
help += "\n\n" help += "\n\n"
help += textwrap.dedent( help += textwrap.dedent("""
"""
You can use these recipes like so: You can use these recipes like so:
\b \b
sqlite-utils convert my.db mytable mycolumn \\ sqlite-utils convert my.db mytable mycolumn \\
'r.jsonsplit(value, delimiter=":")' 'r.jsonsplit(value, delimiter=":")'
""" """).strip()
).strip()
return help return help

View file

@ -15,6 +15,7 @@ from collections.abc import Mapping
import contextlib import contextlib
import datetime import datetime
import decimal import decimal
import importlib
import inspect import inspect
import itertools import itertools
import json import json
@ -22,7 +23,7 @@ import os
import pathlib import pathlib
import re import re
import secrets import secrets
from sqlite_fts4 import rank_bm25 # type: ignore from sqlite_fts4 import rank_bm25
import textwrap import textwrap
from typing import ( from typing import (
cast, cast,
@ -43,7 +44,7 @@ import uuid
from sqlite_utils.plugins import pm from sqlite_utils.plugins import pm
try: try:
from sqlite_dump import iterdump # type: ignore[import-not-found] iterdump = importlib.import_module("sqlite_dump").iterdump
except ImportError: except ImportError:
iterdump = None iterdump = None
@ -82,15 +83,17 @@ def quote_identifier(identifier: str) -> str:
return '"{}"'.format(identifier.replace('"', '""')) return '"{}"'.format(identifier.replace('"', '""'))
pd: Any = None
try: try:
import pandas as pd # type: ignore pd = importlib.import_module("pandas")
except ImportError: except ImportError:
pd = None # type: ignore pd = None
np: Any = None
try: try:
import numpy as np # type: ignore np = importlib.import_module("numpy")
except ImportError: except ImportError:
np = None # type: ignore np = None
Column = namedtuple( Column = namedtuple(
"Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
@ -190,7 +193,10 @@ class Default:
DEFAULT = Default() DEFAULT = Default()
COLUMN_TYPE_MAPPING = { Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None]
COLUMN_TYPE_MAPPING: Dict[Any, str] = {
float: "REAL", float: "REAL",
int: "INTEGER", int: "INTEGER",
bool: "INTEGER", bool: "INTEGER",
@ -339,7 +345,7 @@ class Database:
memory_name: Optional[str] = None, memory_name: Optional[str] = None,
recreate: bool = False, recreate: bool = False,
recursive_triggers: bool = True, recursive_triggers: bool = True,
tracer: Optional[Callable] = None, tracer: Optional[Tracer] = None,
use_counts_table: bool = False, use_counts_table: bool = False,
execute_plugins: bool = True, execute_plugins: bool = True,
use_old_upsert: bool = False, use_old_upsert: bool = False,
@ -375,8 +381,8 @@ class Database:
self.conn = sqlite3.connect(str(filename_or_conn)) self.conn = sqlite3.connect(str(filename_or_conn))
else: else:
assert not recreate, "recreate cannot be used with connections, only paths" assert not recreate, "recreate cannot be used with connections, only paths"
self.conn = filename_or_conn self.conn = cast(sqlite3.Connection, filename_or_conn)
self._tracer = tracer self._tracer: Optional[Tracer] = tracer
if recursive_triggers: if recursive_triggers:
self.execute("PRAGMA recursive_triggers=on;") self.execute("PRAGMA recursive_triggers=on;")
self._registered_functions: set = set() self._registered_functions: set = set()
@ -421,7 +427,7 @@ class Database:
@contextlib.contextmanager @contextlib.contextmanager
def tracer( def tracer(
self, tracer: Optional[Callable[[str, Optional[Sequence]], None]] = None self, tracer: Optional[Tracer] = None
) -> Generator["Database", None, None]: ) -> Generator["Database", None, None]:
""" """
Context manager to temporarily set a tracer function - all executed SQL queries will Context manager to temporarily set a tracer function - all executed SQL queries will
@ -439,7 +445,7 @@ class Database:
:param tracer: Callable accepting ``sql`` and ``parameters`` arguments :param tracer: Callable accepting ``sql`` and ``parameters`` arguments
""" """
prev_tracer = self._tracer prev_tracer = self._tracer
self._tracer = tracer or print self._tracer = tracer or cast(Tracer, print)
try: try:
yield self yield self
finally: finally:
@ -2275,12 +2281,10 @@ class Table(Queryable):
"{}_{}".format(index_name, suffix) if suffix else index_name "{}_{}".format(index_name, suffix) if suffix else index_name
) )
sql = ( sql = (
textwrap.dedent( textwrap.dedent("""
"""
CREATE {unique}INDEX {if_not_exists}{index_name} CREATE {unique}INDEX {if_not_exists}{index_name}
ON {table_name} ({columns}); ON {table_name} ({columns});
""" """)
)
.strip() .strip()
.format( .format(
index_name=quote_identifier(created_index_name), index_name=quote_identifier(created_index_name),
@ -2475,8 +2479,7 @@ class Table(Queryable):
See :ref:`python_api_cached_table_counts` for details. See :ref:`python_api_cached_table_counts` for details.
""" """
sql = ( sql = (
textwrap.dedent( textwrap.dedent("""
"""
{create_counts_table} {create_counts_table}
CREATE TRIGGER IF NOT EXISTS {trigger_insert} AFTER INSERT ON {table} CREATE TRIGGER IF NOT EXISTS {trigger_insert} AFTER INSERT ON {table}
BEGIN BEGIN
@ -2501,8 +2504,7 @@ class Table(Queryable):
); );
END; END;
INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from {table})); INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from {table}));
""" """)
)
.strip() .strip()
.format( .format(
create_counts_table=_COUNTS_TABLE_CREATE_SQL.format( create_counts_table=_COUNTS_TABLE_CREATE_SQL.format(
@ -2554,14 +2556,12 @@ class Table(Queryable):
:param replace: Should any existing FTS index for this table be replaced by the new one? :param replace: Should any existing FTS index for this table be replaced by the new one?
""" """
create_fts_sql = ( create_fts_sql = (
textwrap.dedent( textwrap.dedent("""
"""
CREATE VIRTUAL TABLE {table_fts} USING {fts_version} ( CREATE VIRTUAL TABLE {table_fts} USING {fts_version} (
{columns},{tokenize} {columns},{tokenize}
content={table} content={table}
) )
""" """)
)
.strip() .strip()
.format( .format(
table=quote_identifier(self.name), table=quote_identifier(self.name),
@ -2599,8 +2599,7 @@ class Table(Queryable):
table = quote_identifier(self.name) table = quote_identifier(self.name)
table_fts = quote_identifier(self.name + "_fts") table_fts = quote_identifier(self.name + "_fts")
triggers = ( triggers = (
textwrap.dedent( textwrap.dedent("""
"""
CREATE TRIGGER {table_ai} AFTER INSERT ON {table} BEGIN CREATE TRIGGER {table_ai} AFTER INSERT ON {table} BEGIN
INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols});
END; END;
@ -2611,8 +2610,7 @@ class Table(Queryable):
INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols});
INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols});
END; END;
""" """)
)
.strip() .strip()
.format( .format(
table=table, table=table,
@ -2637,12 +2635,10 @@ class Table(Queryable):
""" """
columns_quoted = ", ".join(quote_identifier(c) for c in columns) columns_quoted = ", ".join(quote_identifier(c) for c in columns)
sql = ( sql = (
textwrap.dedent( textwrap.dedent("""
"""
INSERT INTO {table_fts} (rowid, {columns}) INSERT INTO {table_fts} (rowid, {columns})
SELECT rowid, {columns} FROM {table}; SELECT rowid, {columns} FROM {table};
""" """)
)
.strip() .strip()
.format( .format(
table=quote_identifier(self.name), table=quote_identifier(self.name),
@ -2659,17 +2655,11 @@ class Table(Queryable):
if fts_table: if fts_table:
self.db[fts_table].drop() self.db[fts_table].drop()
# Now delete the triggers that related to that table # Now delete the triggers that related to that table
sql = ( sql = textwrap.dedent("""
textwrap.dedent(
"""
SELECT name FROM sqlite_master SELECT name FROM sqlite_master
WHERE type = 'trigger' WHERE type = 'trigger'
AND (sql LIKE '% INSERT INTO [{}]%' OR sql LIKE '% INSERT INTO "{}"%') AND (sql LIKE '% INSERT INTO [{}]%' OR sql LIKE '% INSERT INTO "{}"%')
""" """).strip().format(fts_table, fts_table)
)
.strip()
.format(fts_table, fts_table)
)
trigger_names = [] trigger_names = []
for row in self.db.execute(sql).fetchall(): for row in self.db.execute(sql).fetchall():
trigger_names.append(row[0]) trigger_names.append(row[0])
@ -2695,8 +2685,7 @@ class Table(Queryable):
def detect_fts(self) -> Optional[str]: def detect_fts(self) -> Optional[str]:
"Detect if table has a corresponding FTS virtual table and return it" "Detect if table has a corresponding FTS virtual table and return it"
sql = textwrap.dedent( sql = textwrap.dedent("""
"""
SELECT name FROM sqlite_master SELECT name FROM sqlite_master
WHERE rootpage = 0 WHERE rootpage = 0
AND ( AND (
@ -2707,8 +2696,7 @@ class Table(Queryable):
AND sql LIKE '%VIRTUAL TABLE%USING FTS%' AND sql LIKE '%VIRTUAL TABLE%USING FTS%'
) )
) )
""" """).strip()
).strip()
args = { args = {
"like": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "like": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name),
"like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name),
@ -2724,13 +2712,9 @@ class Table(Queryable):
"Run the ``optimize`` operation against the associated full-text search index table." "Run the ``optimize`` operation against the associated full-text search index table."
fts_table = self.detect_fts() fts_table = self.detect_fts()
if fts_table is not None: if fts_table is not None:
self.db.execute( self.db.execute("""
"""
INSERT INTO {table} ({table}) VALUES ("optimize"); INSERT INTO {table} ({table}) VALUES ("optimize");
""".strip().format( """.strip().format(table=quote_identifier(fts_table)))
table=quote_identifier(fts_table)
)
)
return self return self
def search_sql( def search_sql(
@ -2768,8 +2752,7 @@ class Table(Queryable):
) )
fts_table_quoted = quote_identifier(fts_table) fts_table_quoted = quote_identifier(fts_table)
virtual_table_using = self.db.table(fts_table).virtual_table_using virtual_table_using = self.db.table(fts_table).virtual_table_using
sql = textwrap.dedent( sql = textwrap.dedent("""
"""
with {original} as ( with {original} as (
select select
rowid, rowid,
@ -2786,8 +2769,7 @@ class Table(Queryable):
order by order by
{order_by} {order_by}
{limit_offset} {limit_offset}
""" """).strip()
).strip()
if virtual_table_using == "FTS5": if virtual_table_using == "FTS5":
rank_implementation = "{}.rank".format(fts_table_quoted) rank_implementation = "{}.rank".format(fts_table_quoted)
else: else:
@ -3517,7 +3499,7 @@ class Table(Queryable):
raise ValueError( raise ValueError(
"When using list-based iteration, the first yielded value must be a list of column name strings" "When using list-based iteration, the first yielded value must be a list of column name strings"
) )
column_names = list(first_record) column_names = cast(List[str], list(first_record))
all_columns = column_names all_columns = column_names
num_columns = len(column_names) num_columns = len(column_names)
# Get the actual first data record # Get the actual first data record
@ -3559,7 +3541,8 @@ class Table(Queryable):
chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk]
column_types = suggest_column_types(chunk_as_dicts) column_types = suggest_column_types(chunk_as_dicts)
else: else:
column_types = suggest_column_types(chunk) # type: ignore[arg-type] dict_chunk = cast(List[Dict[str, Any]], chunk)
column_types = suggest_column_types(dict_chunk)
if extracts: if extracts:
for col in extracts: for col in extracts:
if col in column_types: if col in column_types:
@ -3586,14 +3569,14 @@ class Table(Queryable):
all_columns.insert(0, hash_id) all_columns.insert(0, hash_id)
else: else:
all_columns_set: Set[str] = set() all_columns_set: Set[str] = set()
for record in chunk: for record in cast(List[Dict[str, Any]], chunk):
all_columns_set.update(record.keys()) # type: ignore[union-attr] all_columns_set.update(record.keys())
all_columns = list(sorted(all_columns_set)) all_columns = list(sorted(all_columns_set))
if hash_id: if hash_id:
all_columns.insert(0, hash_id) all_columns.insert(0, hash_id)
else: else:
if not list_mode: if not list_mode:
for record in chunk: for record in cast(List[Dict[str, Any]], chunk):
all_columns += [ all_columns += [
column for column in record if column not in all_columns column for column in record if column not in all_columns
] ]
@ -3791,6 +3774,7 @@ class Table(Queryable):
:param strict: Boolean, apply STRICT mode if creating the table. :param strict: Boolean, apply STRICT mode if creating the table.
""" """
assert isinstance(lookup_values, dict) assert isinstance(lookup_values, dict)
assert pk is not None
if extra_values is not None: if extra_values is not None:
assert isinstance(extra_values, dict) assert isinstance(extra_values, dict)
combined_values = dict(lookup_values) combined_values = dict(lookup_values)
@ -3810,7 +3794,7 @@ class Table(Queryable):
) )
) )
try: try:
return rows[0][pk] # type: ignore[index] return rows[0][pk]
except IndexError: except IndexError:
return self.insert( return self.insert(
combined_values, combined_values,

View file

@ -3,6 +3,7 @@ import contextlib
import csv import csv
import enum import enum
import hashlib import hashlib
import importlib
import io import io
import itertools import itertools
import json import json
@ -21,6 +22,7 @@ from typing import (
Set, Set,
Tuple, Tuple,
Type, Type,
TYPE_CHECKING,
TypeVar, TypeVar,
Union, Union,
cast, cast,
@ -30,22 +32,26 @@ import click
from . import recipes from . import recipes
try: if TYPE_CHECKING:
import pysqlite3 as sqlite3 # type: ignore[import-not-found] # noqa: F401 import sqlite3 # noqa: F401
from pysqlite3 import dbapi2 # type: ignore[import-not-found] # noqa: F401 from sqlite3 import dbapi2 # noqa: F401
OperationalError = dbapi2.OperationalError OperationalError = dbapi2.OperationalError
except ImportError: else:
try: try:
import sqlean as sqlite3 # type: ignore[import-not-found] # noqa: F401 sqlite3 = importlib.import_module("pysqlite3")
from sqlean import dbapi2 # type: ignore[import-not-found] # noqa: F401 dbapi2 = importlib.import_module("pysqlite3.dbapi2")
OperationalError = dbapi2.OperationalError OperationalError = dbapi2.OperationalError
except ImportError: except ImportError:
import sqlite3 # noqa: F401 try:
from sqlite3 import dbapi2 # noqa: F401 sqlite3 = importlib.import_module("sqlean")
dbapi2 = importlib.import_module("sqlean.dbapi2")
OperationalError = dbapi2.OperationalError
except ImportError:
import sqlite3 # noqa: F401
from sqlite3 import dbapi2 # noqa: F401
OperationalError = dbapi2.OperationalError OperationalError = dbapi2.OperationalError
SPATIALITE_PATHS = ( SPATIALITE_PATHS = (
@ -60,7 +66,7 @@ SPATIALITE_PATHS = (
ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit()
# Type alias for row dictionaries - values can be various SQLite-compatible types # Type alias for row dictionaries - values can be various SQLite-compatible types
RowValue = Union[None, int, float, str, bytes, bool] RowValue = Union[None, int, float, str, bytes, bool, List[str]]
Row = Dict[str, RowValue] Row = Dict[str, RowValue]
T = TypeVar("T") T = TypeVar("T")
@ -284,7 +290,7 @@ def _extra_key_strategy(
else: else:
extras_value = row.pop(None) extras_value = row.pop(None)
row_out = cast(Row, row) row_out = cast(Row, row)
row_out[extras_key] = extras_value # type: ignore[assignment] row_out[extras_key] = cast(RowValue, extras_value)
yield row_out yield row_out
@ -464,14 +470,14 @@ class ValueTracker:
def test_integer(self, value: object) -> bool: def test_integer(self, value: object) -> bool:
try: try:
int(value) # type: ignore int(cast(Any, value))
return True return True
except (ValueError, TypeError): except (ValueError, TypeError):
return False return False
def test_float(self, value: object) -> bool: def test_float(self, value: object) -> bool:
try: try:
float(value) # type: ignore[arg-type] float(cast(Any, value))
return True return True
except (ValueError, TypeError): except (ValueError, TypeError):
return False return False

View file

@ -42,14 +42,12 @@ def fresh_db():
@pytest.fixture @pytest.fixture
def existing_db(): def existing_db():
database = Database(memory=True) database = Database(memory=True)
database.executescript( database.executescript("""
"""
CREATE TABLE foo (text TEXT); CREATE TABLE foo (text TEXT);
INSERT INTO foo (text) values ("one"); INSERT INTO foo (text) values ("one");
INSERT INTO foo (text) values ("two"); INSERT INTO foo (text) values ("two");
INSERT INTO foo (text) values ("three"); INSERT INTO foo (text) values ("three");
""" """)
)
return database return database

View file

@ -143,10 +143,7 @@ def db_to_analyze_path(db_to_analyze, tmpdir):
def test_analyze_table(db_to_analyze_path): def test_analyze_table(db_to_analyze_path):
result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path])
assert ( assert result.output.strip() == ("""
result.output.strip()
== (
"""
stuff.id: (1/3) stuff.id: (1/3)
Total rows: 8 Total rows: 8
@ -179,9 +176,7 @@ stuff.size: (3/3)
Most common: Most common:
5: 5 5: 5
3: 4""" 3: 4""").strip()
).strip()
)
def test_analyze_table_save(db_to_analyze_path): def test_analyze_table_save(db_to_analyze_path):

View file

@ -967,12 +967,9 @@ def test_query_json_with_json_cols(db_path):
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, [db_path, "select id, name, friends from dogs"] cli.cli, [db_path, "select id, name, friends from dogs"]
) )
assert ( assert r"""
r"""
[{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}] [{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}]
""".strip() """.strip() == result.output.strip()
== result.output.strip()
)
# With --json-cols: # With --json-cols:
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"] cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"]
@ -1998,12 +1995,10 @@ def test_search_quote(tmpdir):
def test_indexes(tmpdir): def test_indexes(tmpdir):
db_path = str(tmpdir / "test.db") db_path = str(tmpdir / "test.db")
db = Database(db_path) db = Database(db_path)
db.conn.executescript( db.conn.executescript("""
"""
create table Gosh (c1 text, c2 text, c3 text); create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_idx on Gosh(c2, c3 desc); create index Gosh_idx on Gosh(c2, c3 desc);
""" """)
)
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, cli.cli,
["indexes", str(db_path)], ["indexes", str(db_path)],
@ -2094,16 +2089,12 @@ def test_triggers(tmpdir, extra_args, expected):
pk="id", pk="id",
) )
db["counter"].insert({"count": 1}) db["counter"].insert({"count": 1})
db.conn.execute( db.conn.execute(textwrap.dedent("""
textwrap.dedent(
"""
CREATE TRIGGER blah AFTER INSERT ON articles CREATE TRIGGER blah AFTER INSERT ON articles
BEGIN BEGIN
UPDATE counter SET count = count + 1; UPDATE counter SET count = count + 1;
END END
""" """))
)
)
args = ["triggers", db_path] args = ["triggers", db_path]
if extra_args: if extra_args:
args.extend(extra_args) args.extend(extra_args)

View file

@ -371,16 +371,14 @@ def test_convert_multi_complex_column_types(fresh_db_and_path):
], ],
pk="id", pk="id",
) )
code = textwrap.dedent( code = textwrap.dedent("""
"""
if value == 1: if value == 1:
return {"is_str": "", "is_float": 1.2, "is_int": None} return {"is_str": "", "is_float": 1.2, "is_int": None}
elif value == 2: elif value == 2:
return {"is_float": 1, "is_int": 12} return {"is_float": 1, "is_int": 12}
elif value == 3: elif value == 3:
return {"is_bytes": b"blah"} return {"is_bytes": b"blah"}
""" """)
)
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, cli.cli,
[ [

View file

@ -20,7 +20,6 @@ import pathlib
import pytest import pytest
import uuid import uuid
try: try:
import pandas as pd # type: ignore import pandas as pd # type: ignore
except ImportError: except ImportError:

View file

@ -1,6 +1,5 @@
import pytest import pytest
EXAMPLES = [ EXAMPLES = [
("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"),
("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"),

View file

@ -5,14 +5,12 @@ import pytest
def test_duplicate(fresh_db): def test_duplicate(fresh_db):
# Create table using native Sqlite statement: # Create table using native Sqlite statement:
fresh_db.execute( fresh_db.execute("""CREATE TABLE "table1" (
"""CREATE TABLE "table1" (
"text_col" TEXT, "text_col" TEXT,
"real_col" REAL, "real_col" REAL,
"int_col" INTEGER, "int_col" INTEGER,
"bool_col" INTEGER, "bool_col" INTEGER,
"datetime_col" TEXT)""" "datetime_col" TEXT)""")
)
# Insert one row of mock data: # Insert one row of mock data:
dt = datetime.datetime.now() dt = datetime.datetime.now()
data = { data = {

View file

@ -126,9 +126,7 @@ def test_extract_rowid_table(fresh_db):
' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n'
")" ")"
) )
assert ( assert fresh_db.execute("""
fresh_db.execute(
"""
select select
tree.name, tree.name,
common_name_latin_name.common_name, common_name_latin_name.common_name,
@ -136,10 +134,7 @@ def test_extract_rowid_table(fresh_db):
from tree from tree
join common_name_latin_name join common_name_latin_name
on tree.common_name_latin_name_id = common_name_latin_name.id on tree.common_name_latin_name_id = common_name_latin_name.id
""" """).fetchall() == [("Tree 1", "Palm", "Arecaceae")]
).fetchall()
== [("Tree 1", "Palm", "Arecaceae")]
)
def test_reuse_lookup_table(fresh_db): def test_reuse_lookup_table(fresh_db):

View file

@ -109,13 +109,11 @@ def test_table_repr(fresh_db):
def test_indexes(fresh_db): def test_indexes(fresh_db):
fresh_db.executescript( fresh_db.executescript("""
"""
create table Gosh (c1 text, c2 text, c3 text); create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_c1 on Gosh(c1); create index Gosh_c1 on Gosh(c1);
create index Gosh_c2c3 on Gosh(c2, c3); create index Gosh_c2c3 on Gosh(c2, c3);
""" """)
)
assert [ assert [
Index( Index(
seq=0, seq=0,
@ -130,13 +128,11 @@ def test_indexes(fresh_db):
def test_xindexes(fresh_db): def test_xindexes(fresh_db):
fresh_db.executescript( fresh_db.executescript("""
"""
create table Gosh (c1 text, c2 text, c3 text); create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_c1 on Gosh(c1); create index Gosh_c1 on Gosh(c1);
create index Gosh_c2c3 on Gosh(c2, c3 desc); create index Gosh_c2c3 on Gosh(c2, c3 desc);
""" """)
)
assert fresh_db["Gosh"].xindexes == [ assert fresh_db["Gosh"].xindexes == [
XIndex( XIndex(
name="Gosh_c2c3", name="Gosh_c2c3",

View file

@ -638,15 +638,13 @@ def test_transform_with_indexes_errors(fresh_db, transform_params):
def test_transform_with_unique_constraint_implicit_index(fresh_db): def test_transform_with_unique_constraint_implicit_index(fresh_db):
dogs = fresh_db["dogs"] dogs = fresh_db["dogs"]
# Create a table with a UNIQUE constraint on 'name', which creates an implicit index # Create a table with a UNIQUE constraint on 'name', which creates an implicit index
fresh_db.execute( fresh_db.execute("""
"""
CREATE TABLE dogs ( CREATE TABLE dogs (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
name TEXT UNIQUE, name TEXT UNIQUE,
age INTEGER age INTEGER
); );
""" """)
)
dogs.insert({"id": 1, "name": "Cleo", "age": 5}) dogs.insert({"id": 1, "name": "Cleo", "age": 5})
# Attempt to transform the table without modifying 'name' # Attempt to transform the table without modifying 'name'