mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-07-23 09:24:31 +02:00
More type annotations (#697)
* Add comprehensive type annotations - mypy.ini: expanded configuration with module-specific settings - hookspecs.py: type annotations for hook functions - plugins.py: typed get_plugins() return value - recipes.py: full type annotations for parsedate, parsedatetime, jsonsplit - utils.py: extensive type annotations including Row type alias, TypeTracker, ValueTracker, and all utility functions - db.py: type annotations for Database methods (__exit__, ensure_autocommit_off, tracer, register_function, etc.) and Queryable class methods - tests/test_docs.py: updated to match new signature display format * Fix type errors caught by ty check - Add type: ignore comments for external library type stub limitations (csv.reader, click.progressbar, IOBase.name, Callable.__name__) - Change Iterable to Sequence for SQL where_args parameters - Use db.table() instead of db[name] for proper Table return type - Fix rebuild_fts return type from None to Table - Update test_tracer to expect fewer queries (optimization side effect) * Fix mypy type errors - Add type: ignore comments for runtime-valid patterns mypy can't verify - Fix new_column_types annotation to Dict[str, Set[type]] - Add type: ignore for Default sentinel values passed to create_table * mypy skip tests directory * Fix CI: exclude typing imports from recipe docs, skip mypy on tests - Add Callable and Optional to exclusion list in _generate_convert_help() - Regenerate docs/cli-reference.rst with cog - Add [mypy-tests.*] ignore_errors = True to skip test type errors --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Simon Willison
GitHub
parent
871038505b
commit
8d74ffc932
10 changed files with 274 additions and 154 deletions
|
|
@ -8,7 +8,23 @@ import itertools
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import Dict, cast, BinaryIO, Iterable, Iterator, Optional, Tuple, Type
|
||||
from typing import (
|
||||
Any,
|
||||
BinaryIO,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
import click
|
||||
|
||||
|
|
@ -43,18 +59,24 @@ SPATIALITE_PATHS = (
|
|||
# Mainly so we can restore it if needed in the tests:
|
||||
ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit()
|
||||
|
||||
# Type alias for row dictionaries - values can be various SQLite-compatible types
|
||||
RowValue = Union[None, int, float, str, bytes, bool]
|
||||
Row = Dict[str, RowValue]
|
||||
|
||||
class _CloseableIterator(Iterator[dict]):
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class _CloseableIterator(Iterator[Row]):
|
||||
"""Iterator wrapper that closes a file when iteration is complete."""
|
||||
|
||||
def __init__(self, iterator: Iterator[dict], closeable: io.IOBase):
|
||||
def __init__(self, iterator: Iterator[Row], closeable: io.IOBase) -> None:
|
||||
self._iterator = iterator
|
||||
self._closeable = closeable
|
||||
|
||||
def __iter__(self) -> "_CloseableIterator":
|
||||
return self
|
||||
|
||||
def __next__(self) -> dict:
|
||||
def __next__(self) -> Row:
|
||||
try:
|
||||
return next(self._iterator)
|
||||
except StopIteration:
|
||||
|
|
@ -65,7 +87,7 @@ class _CloseableIterator(Iterator[dict]):
|
|||
self._closeable.close()
|
||||
|
||||
|
||||
def maximize_csv_field_size_limit():
|
||||
def maximize_csv_field_size_limit() -> None:
|
||||
"""
|
||||
Increase the CSV field size limit to the maximum possible.
|
||||
"""
|
||||
|
|
@ -108,20 +130,25 @@ def find_spatialite() -> Optional[str]:
|
|||
return None
|
||||
|
||||
|
||||
def suggest_column_types(records):
|
||||
all_column_types = {}
|
||||
def suggest_column_types(
|
||||
records: Iterable[Dict[str, Any]],
|
||||
) -> Dict[str, type]:
|
||||
all_column_types: Dict[str, Set[type]] = {}
|
||||
for record in records:
|
||||
for key, value in record.items():
|
||||
all_column_types.setdefault(key, set()).add(type(value))
|
||||
return types_for_column_types(all_column_types)
|
||||
|
||||
|
||||
def types_for_column_types(all_column_types):
|
||||
column_types = {}
|
||||
def types_for_column_types(
|
||||
all_column_types: Dict[str, Set[type]],
|
||||
) -> Dict[str, type]:
|
||||
column_types: Dict[str, type] = {}
|
||||
for key, types in all_column_types.items():
|
||||
# Ignore null values if at least one other type present:
|
||||
if len(types) > 1:
|
||||
types.discard(None.__class__)
|
||||
t: type
|
||||
if {None.__class__} == types:
|
||||
t = str
|
||||
elif len(types) == 1:
|
||||
|
|
@ -143,7 +170,7 @@ def types_for_column_types(all_column_types):
|
|||
return column_types
|
||||
|
||||
|
||||
def column_affinity(column_type):
|
||||
def column_affinity(column_type: str) -> type:
|
||||
# Implementation of SQLite affinity rules from
|
||||
# https://www.sqlite.org/datatype3.html#determination_of_column_affinity
|
||||
assert isinstance(column_type, str)
|
||||
|
|
@ -162,38 +189,42 @@ def column_affinity(column_type):
|
|||
return float
|
||||
|
||||
|
||||
def decode_base64_values(doc):
|
||||
def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
|
||||
to_fix = [
|
||||
k
|
||||
for k in doc
|
||||
if isinstance(doc[k], dict)
|
||||
and doc[k].get("$base64") is True
|
||||
and "encoded" in doc[k]
|
||||
and cast(dict, doc[k]).get("$base64") is True
|
||||
and "encoded" in cast(dict, doc[k])
|
||||
]
|
||||
if not to_fix:
|
||||
return doc
|
||||
return dict(doc, **{k: base64.b64decode(doc[k]["encoded"]) for k in to_fix})
|
||||
return dict(
|
||||
doc, **{k: base64.b64decode(cast(dict, doc[k])["encoded"]) for k in to_fix}
|
||||
)
|
||||
|
||||
|
||||
class UpdateWrapper:
|
||||
def __init__(self, wrapped, update):
|
||||
def __init__(self, wrapped: io.IOBase, update: Callable[[int], None]) -> None:
|
||||
self._wrapped = wrapped
|
||||
self._update = update
|
||||
|
||||
def __iter__(self):
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
for line in self._wrapped:
|
||||
self._update(len(line))
|
||||
yield line
|
||||
|
||||
def read(self, size=-1):
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
data = self._wrapped.read(size)
|
||||
self._update(len(data))
|
||||
return data
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def file_progress(file, silent=False, **kwargs):
|
||||
def file_progress(
|
||||
file: io.IOBase, silent: bool = False, **kwargs: object
|
||||
) -> Generator[Union[io.IOBase, "UpdateWrapper"], None, None]:
|
||||
if silent:
|
||||
yield file
|
||||
return
|
||||
|
|
@ -206,8 +237,8 @@ def file_progress(file, silent=False, **kwargs):
|
|||
if fileno == 0: # 0 means stdin
|
||||
yield file
|
||||
else:
|
||||
file_length = os.path.getsize(file.name)
|
||||
with click.progressbar(length=file_length, **kwargs) as bar:
|
||||
file_length = os.path.getsize(file.name) # type: ignore
|
||||
with click.progressbar(length=file_length, **kwargs) as bar: # type: ignore
|
||||
yield UpdateWrapper(file, bar.update)
|
||||
|
||||
|
||||
|
|
@ -231,28 +262,30 @@ class RowError(Exception):
|
|||
|
||||
|
||||
def _extra_key_strategy(
|
||||
reader: Iterable[dict],
|
||||
reader: Iterable[Dict[Optional[str], object]],
|
||||
ignore_extras: Optional[bool] = False,
|
||||
extras_key: Optional[str] = None,
|
||||
) -> Iterable[dict]:
|
||||
) -> Iterable[Row]:
|
||||
# Logic for handling CSV rows with more values than there are headings
|
||||
for row in reader:
|
||||
# DictReader adds a 'None' key with extra row values
|
||||
if None not in row:
|
||||
yield row
|
||||
yield cast(Row, row)
|
||||
elif ignore_extras:
|
||||
# ignoring row.pop(none) because of this issue:
|
||||
# https://github.com/simonw/sqlite-utils/issues/440#issuecomment-1155358637
|
||||
row.pop(None) # type: ignore
|
||||
yield row
|
||||
row.pop(None)
|
||||
yield cast(Row, row)
|
||||
elif not extras_key:
|
||||
extras = row.pop(None) # type: ignore
|
||||
extras = row.pop(None)
|
||||
raise RowError(
|
||||
"Row {} contained these extra values: {}".format(row, extras)
|
||||
)
|
||||
else:
|
||||
row[extras_key] = row.pop(None) # type: ignore
|
||||
yield row
|
||||
extras_value = row.pop(None)
|
||||
row_out = cast(Row, row)
|
||||
row_out[extras_key] = extras_value # type: ignore[assignment]
|
||||
yield row_out
|
||||
|
||||
|
||||
def rows_from_file(
|
||||
|
|
@ -262,7 +295,7 @@ def rows_from_file(
|
|||
encoding: Optional[str] = None,
|
||||
ignore_extras: Optional[bool] = False,
|
||||
extras_key: Optional[str] = None,
|
||||
) -> Tuple[Iterable[dict], Format]:
|
||||
) -> Tuple[Iterable[Row], Format]:
|
||||
"""
|
||||
Load a sequence of dictionaries from a file-like object containing one of four different formats.
|
||||
|
||||
|
|
@ -324,10 +357,17 @@ def rows_from_file(
|
|||
rows = _extra_key_strategy(reader, ignore_extras, extras_key)
|
||||
return _CloseableIterator(iter(rows), decoded_fp), Format.CSV
|
||||
elif format == Format.TSV:
|
||||
rows = rows_from_file(
|
||||
rows, _ = rows_from_file(
|
||||
fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
|
||||
)[0]
|
||||
return _extra_key_strategy(rows, ignore_extras, extras_key), Format.TSV
|
||||
)
|
||||
return (
|
||||
_extra_key_strategy(
|
||||
cast(Iterable[Dict[Optional[str], object]], rows),
|
||||
ignore_extras,
|
||||
extras_key,
|
||||
),
|
||||
Format.TSV,
|
||||
)
|
||||
elif format is None:
|
||||
# Detect the format, then call this recursively
|
||||
buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
|
||||
|
|
@ -349,8 +389,15 @@ def rows_from_file(
|
|||
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
|
||||
)
|
||||
# Make sure we return the format we detected
|
||||
format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
|
||||
return _extra_key_strategy(rows, ignore_extras, extras_key), format
|
||||
detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
|
||||
return (
|
||||
_extra_key_strategy(
|
||||
cast(Iterable[Dict[Optional[str], object]], rows),
|
||||
ignore_extras,
|
||||
extras_key,
|
||||
),
|
||||
detected_format,
|
||||
)
|
||||
else:
|
||||
raise RowsFromFileError("Bad format")
|
||||
|
||||
|
|
@ -376,10 +423,10 @@ class TypeTracker:
|
|||
db["creatures"].transform(types=tracker.types)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.trackers = {}
|
||||
def __init__(self) -> None:
|
||||
self.trackers: Dict[str, "ValueTracker"] = {}
|
||||
|
||||
def wrap(self, iterator: Iterable[dict]) -> Iterable[dict]:
|
||||
def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
|
||||
"""
|
||||
Use this to loop through an existing iterator, tracking the column types
|
||||
as part of the iteration.
|
||||
|
|
@ -402,27 +449,29 @@ class TypeTracker:
|
|||
|
||||
|
||||
class ValueTracker:
|
||||
def __init__(self):
|
||||
couldbe: Dict[str, Callable[[object], bool]]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()}
|
||||
|
||||
@classmethod
|
||||
def get_tests(cls):
|
||||
def get_tests(cls) -> List[str]:
|
||||
return [
|
||||
key.split("test_")[-1]
|
||||
for key in cls.__dict__.keys()
|
||||
if key.startswith("test_")
|
||||
]
|
||||
|
||||
def test_integer(self, value):
|
||||
def test_integer(self, value: object) -> bool:
|
||||
try:
|
||||
int(value)
|
||||
int(value) # type: ignore
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
def test_float(self, value):
|
||||
def test_float(self, value: object) -> bool:
|
||||
try:
|
||||
float(value)
|
||||
float(value) # type: ignore[arg-type]
|
||||
return True
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
|
@ -431,7 +480,7 @@ class ValueTracker:
|
|||
return self.guessed_type + ": possibilities = " + repr(self.couldbe)
|
||||
|
||||
@property
|
||||
def guessed_type(self):
|
||||
def guessed_type(self) -> str:
|
||||
options = set(self.couldbe.keys())
|
||||
# Return based on precedence
|
||||
for key in self.get_tests():
|
||||
|
|
@ -439,10 +488,10 @@ class ValueTracker:
|
|||
return key
|
||||
return "text"
|
||||
|
||||
def evaluate(self, value):
|
||||
def evaluate(self, value: object) -> None:
|
||||
if not value or not self.couldbe:
|
||||
return
|
||||
not_these = []
|
||||
not_these: List[str] = []
|
||||
for name, test in self.couldbe.items():
|
||||
if not test(value):
|
||||
not_these.append(name)
|
||||
|
|
@ -451,45 +500,47 @@ class ValueTracker:
|
|||
|
||||
|
||||
class NullProgressBar:
|
||||
def __init__(self, *args):
|
||||
def __init__(self, *args: Iterable[T]) -> None:
|
||||
self.args = args
|
||||
|
||||
def __iter__(self):
|
||||
yield from self.args[0]
|
||||
def __iter__(self) -> Iterator[T]:
|
||||
yield from self.args[0] # type: ignore
|
||||
|
||||
def update(self, value):
|
||||
def update(self, value: int) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def progressbar(*args, **kwargs):
|
||||
def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None]:
|
||||
silent = kwargs.pop("silent")
|
||||
if silent:
|
||||
yield NullProgressBar(*args)
|
||||
else:
|
||||
with click.progressbar(*args, **kwargs) as bar:
|
||||
with click.progressbar(*args, **kwargs) as bar: # type: ignore
|
||||
yield bar
|
||||
|
||||
|
||||
def _compile_code(code, imports, variable="value"):
|
||||
globals = {"r": recipes, "recipes": recipes}
|
||||
def _compile_code(
|
||||
code: str, imports: Iterable[str], variable: str = "value"
|
||||
) -> Callable[..., Any]:
|
||||
globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes}
|
||||
# Handle imports first so they're available for all approaches
|
||||
for import_ in imports:
|
||||
globals[import_.split(".")[0]] = __import__(import_)
|
||||
globals_dict[import_.split(".")[0]] = __import__(import_)
|
||||
|
||||
# If user defined a convert() function, return that
|
||||
try:
|
||||
exec(code, globals)
|
||||
return globals["convert"]
|
||||
exec(code, globals_dict)
|
||||
return cast(Callable[..., object], globals_dict["convert"])
|
||||
except (AttributeError, SyntaxError, NameError, KeyError, TypeError):
|
||||
pass
|
||||
|
||||
# Check if code is a direct callable reference
|
||||
# e.g. "r.parsedate" instead of "r.parsedate(value)"
|
||||
try:
|
||||
fn = eval(code, globals)
|
||||
fn = eval(code, globals_dict)
|
||||
if callable(fn):
|
||||
return fn
|
||||
return cast(Callable[..., object], fn)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -514,11 +565,11 @@ def _compile_code(code, imports, variable="value"):
|
|||
if code_o is None:
|
||||
raise SyntaxError("Could not compile code")
|
||||
|
||||
exec(code_o, globals)
|
||||
return globals["fn"]
|
||||
exec(code_o, globals_dict)
|
||||
return cast(Callable[..., object], globals_dict["fn"])
|
||||
|
||||
|
||||
def chunks(sequence: Iterable, size: int) -> Iterable[Iterable]:
|
||||
def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]:
|
||||
"""
|
||||
Iterate over chunks of the sequence of the given size.
|
||||
|
||||
|
|
@ -530,7 +581,7 @@ def chunks(sequence: Iterable, size: int) -> Iterable[Iterable]:
|
|||
yield itertools.chain([item], itertools.islice(iterator, size - 1))
|
||||
|
||||
|
||||
def hash_record(record: Dict, keys: Optional[Iterable[str]] = None):
|
||||
def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str:
|
||||
"""
|
||||
``record`` should be a Python dictionary. Returns a sha1 hash of the
|
||||
keys and values in that record.
|
||||
|
|
@ -551,7 +602,7 @@ def hash_record(record: Dict, keys: Optional[Iterable[str]] = None):
|
|||
:param record: Record to generate a hash for
|
||||
:param keys: Subset of keys to use for that hash
|
||||
"""
|
||||
to_hash = record
|
||||
to_hash: Dict[str, Any] = record
|
||||
if keys is not None:
|
||||
to_hash = {key: record[key] for key in keys}
|
||||
return hashlib.sha1(
|
||||
|
|
@ -561,7 +612,7 @@ def hash_record(record: Dict, keys: Optional[Iterable[str]] = None):
|
|||
).hexdigest()
|
||||
|
||||
|
||||
def _flatten(d):
|
||||
def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
|
||||
for key, value in d.items():
|
||||
if isinstance(value, dict):
|
||||
for key2, value2 in _flatten(value):
|
||||
|
|
@ -570,7 +621,7 @@ def _flatten(d):
|
|||
yield key, value
|
||||
|
||||
|
||||
def flatten(row: dict) -> dict:
|
||||
def flatten(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}``
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue