diff --git a/.gitignore b/.gitignore index 5b5d2c6..6743708 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ venv .schema .vscode .hypothesis -.claude/ Pipfile Pipfile.lock uv.lock diff --git a/Justfile b/Justfile index be41523..5caa120 100644 --- a/Justfile +++ b/Justfile @@ -16,7 +16,6 @@ uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt - uv run --group docs codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt # Rebuild docs with cog @cog: diff --git a/docs/changelog.rst b/docs/changelog.rst index a853aa2..5b9355f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,20 +4,6 @@ Changelog =========== -.. _v3_39_1: - -3.39.1 (2026-07-25) -------------------- - -- Fixed a bug where ``table.delete_where()`` left the connection in an open transaction, causing deleted rows to be silently restored when the connection was closed. (:issue:`815`) - -.. _v4_1_1: - -4.1.1 (2026-07-12) ------------------- - -- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) -- The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) .. _v4_1: 4.1 (2026-07-11) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a4ec402..9fafe28 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -662,7 +662,7 @@ See :ref:`cli_convert`. Convert a string like a,b,c into a JSON array ["a", "b", "c"] r.parsedate(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = False, - errors: 'object | None' = None) -> 'str | None' + errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a date and convert it to ISO date format: yyyy-mm-dd - dayfirst=True: treat xx as the day in xx/yy/zz @@ -671,7 +671,7 @@ See :ref:`cli_convert`. - errors=r.SET_NULL to set values that cannot be parsed to null r.parsedatetime(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = - False, errors: 'object | None' = None) -> 'str | None' + False, errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS - dayfirst=True: treat xx as the day in xx/yy/zz diff --git a/docs/conf.py b/docs/conf.py index 62d4642..4f29b39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + import inspect -import sys from pathlib import Path -from subprocess import PIPE, CalledProcessError, Popen, check_output +from subprocess import Popen, PIPE, check_output +import sys # This file is execfile()d with the current directory set to its # containing dir. @@ -47,7 +50,7 @@ extlinks = { def _linkcode_git_ref(): try: return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() - except (CalledProcessError, OSError): + except Exception: return "main" @@ -76,7 +79,7 @@ def linkcode_resolve(domain, info): obj = inspect.unwrap(obj) source_file = inspect.getsourcefile(obj) _, line_number = inspect.getsourcelines(obj) - except (OSError, TypeError, ValueError): + except Exception: return None if source_file is None: diff --git a/docs/python-api.rst b/docs/python-api.rst index 43b734d..1ed238e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -434,10 +434,9 @@ The library will never commit a transaction you opened. If you call write method Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. -Some related safeguards to be aware of: +Two related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. -- ``table.transform()`` raises a ``sqlite_utils.db.TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions, because the pragma cannot be turned off mid-transaction to protect those referencing rows - see :ref:`python_api_transform_foreign_keys_transactions`. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. .. _python_api_transactions_modes: @@ -1997,36 +1996,6 @@ If you want to do something more advanced, you can call the ``table.transform_sq This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. -.. _python_api_transform_foreign_keys_transactions: - -Foreign keys and transactions ------------------------------ - -Because ``.transform()`` drops the old table, running it with ``PRAGMA foreign_keys`` enabled could fire ``ON DELETE`` actions on any tables that reference it - an inbound ``ON DELETE CASCADE`` foreign key would silently delete those referencing rows. To prevent this, ``.transform()`` turns ``PRAGMA foreign_keys`` off for the duration of the operation and restores it afterwards, running ``PRAGMA foreign_key_check`` before committing. - -``PRAGMA foreign_keys`` cannot be changed inside a transaction, so this protection is impossible if you call ``.transform()`` while a transaction is already open - for example inside a ``with db.atomic():`` block or after ``db.begin()``. If ``PRAGMA foreign_keys`` is on and another table references the table being transformed with a destructive ``ON DELETE`` action - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT`` - the method will refuse to run and raise a ``sqlite_utils.db.TransactionError``: - -.. code-block:: python - - from sqlite_utils.db import TransactionError - - try: - with db.atomic(): - db["authors"].transform(types={"id": str}) - except TransactionError as ex: - print("Could not transform in transaction:", ex) - -To transform such a table either call ``.transform()`` outside of the transaction, or execute ``PRAGMA foreign_keys = off`` before opening it: - -.. code-block:: python - - db.execute("PRAGMA foreign_keys = off") - with db.atomic(): - db["authors"].transform(types={"id": str}) - db.execute("PRAGMA foreign_keys = on") - -Tables referenced by foreign keys without a destructive action (the default ``NO ACTION``, or ``RESTRICT``) can still be transformed inside a transaction - sqlite-utils uses ``PRAGMA defer_foreign_keys`` to postpone the foreign key checks until the transaction commits. - .. _python_api_extract: Extracting columns into a separate table diff --git a/pyproject.toml b/pyproject.toml index 6bc0a64..971f5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1.1" +version = "4.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -79,14 +79,7 @@ build-backend = "setuptools.build_meta" max-line-length = 160 # Black compatibility, E203 whitespace before ':': extend-ignore = ["E203"] -extend-exclude = [ - ".venv", - ".claude", - "build", - "dist", - "docs", - "sqlite_utils.egg-info", -] +extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"] [tool.setuptools.package-data] sqlite_utils = ["py.typed"] diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 0d25716..58ee7ab 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,6 +1,7 @@ -from .db import Database -from .hookspecs import hookimpl, hookspec -from .migrations import Migrations from .utils import suggest_column_types +from .hookspecs import hookimpl +from .hookspecs import hookspec +from .db import Database +from .migrations import Migrations -__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] +__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index dab4b67..e0b8969 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,30 +1,17 @@ import base64 -import csv as csv_std import difflib -import hashlib -import inspect -import io -import itertools -import json -import os -import pathlib -import pdb # noqa: T100 -import sys -import textwrap -from datetime import datetime, timezone -from runpy import run_module from typing import Any - import click -import tabulate from click_default_group import DefaultGroup - +from datetime import datetime, timezone +import hashlib +import pathlib +from runpy import run_module import sqlite_utils -from sqlite_utils import recipes from sqlite_utils.db import ( - DEFAULT, AlterError, BadMultiValues, + DEFAULT, DescIndex, InvalidColumns, NoTable, @@ -32,28 +19,36 @@ from sqlite_utils.db import ( PrimaryKeyRequired, quote_identifier, ) -from sqlite_utils.plugins import ensure_plugins_loaded, get_plugins, pm +from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins from sqlite_utils.utils import maximize_csv_field_size_limit - +from sqlite_utils import recipes +import textwrap +import inspect +import io +import itertools +import json +import os +import pdb +import sys +import csv as csv_std +import tabulate from .utils import ( - Format, OperationalError, - TypeTracker, _compile_code, chunks, - decode_base64_values, dedupe_keys, file_progress, find_spatialite, + flatten as _flatten, + sqlite3, + decode_base64_values, progressbar, rows_from_file, - sqlite3, -) -from .utils import ( - flatten as _flatten, + Format, + TypeTracker, ) -CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} +CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) def _register_db_for_cleanup(db): @@ -72,7 +67,7 @@ def _close_databases(ctx): for db in ctx.meta.get("_databases_to_close", []): try: db.close() - except sqlite3.Error: + except Exception: pass @@ -179,6 +174,7 @@ def functions_option(fn): @click.version_option() def cli(): "Commands for interacting with a SQLite database" + pass @cli.command() @@ -895,7 +891,7 @@ def enable_counts(path, tables, load_extension): # Check all tables exist bad_tables = [table for table in tables if not db[table].exists()] if bad_tables: - raise click.ClickException(f"Invalid tables: {bad_tables}") + raise click.ClickException("Invalid tables: {}".format(bad_tables)) for table in tables: db.table(table).enable_counts() @@ -1144,7 +1140,9 @@ def insert_upsert_implementation( ) ): raise click.ClickException( - f"{e.args[0]}\n\nTry using --alter to add additional columns" + "{}\n\nTry using --alter to add additional columns".format( + e.args[0] + ) ) # If we can find sql= and parameters= arguments, show those variables = _find_variables(e.__traceback__, ["sql", "parameters"]) @@ -1242,7 +1240,7 @@ def insert_upsert_implementation( reader = csv_std.reader(decoded, **csv_reader_args) # type: ignore first_row = next(reader) if no_headers: - headers = [f"untitled_{i + 1}" for i in range(len(first_row))] + headers = ["untitled_{}".format(i + 1) for i in range(len(first_row))] reader = itertools.chain([first_row], reader) else: headers = first_row @@ -1271,7 +1269,9 @@ def insert_upsert_implementation( docs = [docs] except json.decoder.JSONDecodeError as ex: raise click.ClickException( - f"Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {ex}" + "Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {}".format( + ex + ) ) if flatten: docs = (_flatten(doc) for doc in docs) @@ -1290,7 +1290,7 @@ def insert_upsert_implementation( docs = (fn(doc["line"]) for doc in docs) elif text: # Special case: this is allowed to be an iterable - text_value = next(iter(docs))["text"] + text_value = list(docs)[0]["text"] fn_return = fn(text_value) if isinstance(fn_return, dict): docs = [fn_return] @@ -1774,14 +1774,17 @@ def create_table( ctype = columns.pop(0) if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - f"column types must be one of {VALID_COLUMN_TYPES}" + "column types must be one of {}".format(VALID_COLUMN_TYPES) ) coltypes[name] = ctype.upper() # Does table already exist? - if table in db.table_names() and not ignore and not replace and not transform: - raise click.ClickException( - f'Table "{table}" already exists. Use --replace to delete and replace it.' - ) + if table in db.table_names(): + if not ignore and not replace and not transform: + raise click.ClickException( + 'Table "{}" already exists. Use --replace to delete and replace it.'.format( + table + ) + ) db.table(table).create( coltypes, pk=pks[0] if len(pks) == 1 else pks, @@ -1816,7 +1819,7 @@ def duplicate(path, table, new_table, ignore, load_extension): db.table(table).duplicate(new_table) except NoTable: if not ignore: - raise click.ClickException(f'Table "{table}" does not exist') + raise click.ClickException('Table "{}" does not exist'.format(table)) @cli.command(name="rename-table") @@ -1840,7 +1843,9 @@ def rename_table(path, table, new_name, ignore, load_extension): db.rename_table(table, new_name) except sqlite3.OperationalError as ex: if not ignore: - raise click.ClickException(f'Table "{table}" could not be renamed. {ex!s}') + raise click.ClickException( + 'Table "{}" could not be renamed. {}'.format(table, str(ex)) + ) @cli.command(name="drop-table") @@ -1869,10 +1874,10 @@ def drop_table(path, table, ignore, load_extension): # A view exists with this name if not ignore: raise click.ClickException( - f'"{table}" is a view, not a table - use drop-view to drop it' + '"{}" is a view, not a table - use drop-view to drop it'.format(table) ) except OperationalError: - raise click.ClickException(f'Table "{table}" does not exist') + raise click.ClickException('Table "{}" does not exist'.format(table)) @cli.command(name="create-view") @@ -1914,7 +1919,9 @@ def create_view(path, view, select, ignore, replace, load_extension): db.view(view).drop() else: raise click.ClickException( - f'View "{view}" already exists. Use --replace to delete and replace it.' + 'View "{}" already exists. Use --replace to delete and replace it.'.format( + view + ) ) db.create_view(view, select) @@ -1946,9 +1953,9 @@ def drop_view(path, view, ignore, load_extension): return if view in db.table_names(): raise click.ClickException( - f'"{view}" is a table, not a view - use drop-table to drop it' + '"{}" is a table, not a view - use drop-table to drop it'.format(view) ) - raise click.ClickException(f'View "{view}" does not exist') + raise click.ClickException('View "{}" does not exist'.format(view)) @cli.command() @@ -2170,7 +2177,7 @@ def memory( file_path = pathlib.Path(path) stem = file_path.stem if stem_counts.get(stem): - file_table = f"{stem}_{stem_counts[stem]}" + file_table = "{}_{}".format(stem, stem_counts[stem]) else: file_table = stem stem_counts[stem] = stem_counts.get(stem, 1) + 1 @@ -2189,14 +2196,14 @@ def memory( if tracker is not None and db.table(file_table).exists(): db.table(file_table).transform(types=tracker.types) # Add convenient t / t1 / t2 views - view_names = [f"t{i + 1}"] + view_names = ["t{}".format(i + 1)] if i == 0: view_names.append("t") for view_name in view_names: if not db[view_name].exists(): db.create_view( view_name, - f"select * from {quote_identifier(file_table)}", + "select * from {}".format(quote_identifier(file_table)), ) finally: if should_close_fp and fp: @@ -2366,17 +2373,19 @@ def search( # Check table exists table_obj = db.table(dbtable) if not table_obj.exists(): - raise click.ClickException(f"Table '{dbtable}' does not exist") + raise click.ClickException("Table '{}' does not exist".format(dbtable)) if not table_obj.detect_fts(): raise click.ClickException( - f"Table '{dbtable}' is not configured for full-text search" + "Table '{}' is not configured for full-text search".format(dbtable) ) if column: # Check they all exist table_columns = table_obj.columns_dict for c in column: if c not in table_columns: - raise click.ClickException(f"Table '{dbtable}' has no column '{c}") + raise click.ClickException( + "Table '{}' has no column '{}".format(dbtable, c) + ) sql = table_obj.search_sql(columns=column, order_by=order, limit=limit) if show_sql: click.echo(sql) @@ -2403,7 +2412,7 @@ def search( except click.ClickException as e: if "malformed MATCH expression" in str(e) or "unterminated string" in str(e): raise click.ClickException( - f"{e!s}\n\nTry running this again with the --quote option" + "{}\n\nTry running this again with the --quote option".format(str(e)) ) else: raise @@ -2470,15 +2479,15 @@ def rows( columns = "*" if column: columns = ", ".join(quote_identifier(c) for c in column) - sql = f"select {columns} from {quote_identifier(dbtable)}" + sql = "select {} from {}".format(columns, quote_identifier(dbtable)) if where: sql += " where " + where if order: sql += " order by " + order if limit: - sql += f" limit {limit}" + sql += " limit {}".format(limit) if offset: - sql += f" offset {offset}" + sql += " offset {}".format(offset) ctx.invoke( query, path=path, @@ -2751,7 +2760,7 @@ def transform( for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - f"column types must be one of {VALID_COLUMN_TYPES}" + "column types must be one of {}".format(VALID_COLUMN_TYPES) ) types[column] = ctype.upper() @@ -2849,12 +2858,12 @@ def extract( db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) - kwargs: dict[str, Any] = { - "columns": columns, - "table": other_table, - "fk_column": fk_column, - "rename": dict(rename), - } + kwargs: dict[str, Any] = dict( + columns=columns, + table=other_table, + fk_column=fk_column, + rename=dict(rename), + ) try: db.table(table).extract(**kwargs) except (NoTable, InvalidColumns) as e: @@ -2949,7 +2958,7 @@ def insert_files( with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): - for file_path, relative_path in bar: + for path, relative_path in bar: row = {} # content_text is special case as it considers 'encoding' @@ -2961,21 +2970,19 @@ def insert_files( raise UnicodeDecodeErrorForPath(e, resolved) lookups = dict(FILE_COLUMNS, content_text=_content_text) - if file_path == "-": + if path == "-": stdin_data = sys.stdin.buffer.read() # We only support a subset of columns for this case lookups = { "name": lambda p: name or "-", "path": lambda p: name or "-", - "content": lambda p, data=stdin_data: data, - "content_text": lambda p, data=stdin_data: data.decode( + "content": lambda p: stdin_data, + "content_text": lambda p: stdin_data.decode( encoding or "utf-8" ), - "sha256": lambda p, data=stdin_data: hashlib.sha256( - data - ).hexdigest(), - "md5": lambda p, data=stdin_data: hashlib.md5(data).hexdigest(), - "size": lambda p, data=stdin_data: len(data), + "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), + "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), + "size": lambda p: len(stdin_data), } for coldef in column: if ":" in coldef: @@ -2983,7 +2990,7 @@ def insert_files( else: colname, coltype = coldef, coldef try: - value = lookups[coltype](file_path) + value = lookups[coltype](path) row[colname] = value except KeyError: raise click.ClickException( @@ -3011,7 +3018,7 @@ def insert_files( except UnicodeDecodeErrorForPath as e: raise click.ClickException( UNICODE_ERROR.format( - f"Could not read file '{e.path}' as text\n\n{e.exception}" + "Could not read file '{}' as text\n\n{}".format(e.path, e.exception) ) ) @@ -3189,7 +3196,7 @@ def _generate_convert_help(): for name in recipe_names: fn = getattr(recipes, name) doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += f"\n\nr.{name}{inspect.signature(fn)!s}\n\n\b{doc}" + help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: @@ -3292,7 +3299,7 @@ def convert( """.format( column=columns[0], table=table, - where=f" where {where}" if where is not None else "", + where=" where {}".format(where) if where is not None else "", ) for row in db.conn.execute(sql, where_args).fetchall(): click.echo(str(row[0])) @@ -3312,7 +3319,7 @@ def convert( def wrapped_fn(value): try: return fn_(value) - except Exception as ex: # noqa: BLE001 + except Exception as ex: print("\nException raised, dropping into pdb...:", ex) pdb.post_mortem(ex.__traceback__) sys.exit(1) @@ -3332,7 +3339,9 @@ def convert( ) except BadMultiValues as e: raise click.ClickException( - f"When using --multi code must return a Python dictionary - returned: {e.values!r}" + "When using --multi code must return a Python dictionary - returned: {}".format( + repr(e.values) + ) ) @@ -3450,7 +3459,7 @@ def create_spatial_index(db_path, table, column_name, load_extension): def _find_migration_files(migrations): if not migrations: - migrations = [pathlib.Path.cwd()] + migrations = [pathlib.Path(".").resolve()] files = set() for path_str in migrations: path = pathlib.Path(path_str) @@ -3475,7 +3484,7 @@ def _load_migration_sets(files): "__file__": str(filepath), "__name__": "__sqlite_utils_migration__", } - exec(code, namespace) # noqa: S102 + exec(code, namespace) migration_sets.extend( obj for obj in namespace.values() if _compatible_migration_set(obj) ) @@ -3484,17 +3493,17 @@ def _load_migration_sets(files): def _display_migration_list(db, migration_sets): for migration_set in migration_sets: - click.echo(f"Migrations for: {migration_set.name}") + click.echo("Migrations for: {}".format(migration_set.name)) click.echo() click.echo(" Applied:") for migration in migration_set.applied(db): - click.echo(f" {migration.name} - {migration.applied_at}") + click.echo(" {} - {}".format(migration.name, migration.applied_at)) click.echo() click.echo(" Pending:") output = False for migration in migration_set.pending(db): output = True - click.echo(f" {migration.name}") + click.echo(" {}".format(migration.name)) if not output: click.echo(" (none)") click.echo() @@ -3574,7 +3583,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): prev_schema = db.schema if verbose: - click.echo(f"Migrating {db_path}") + click.echo("Migrating {}".format(db_path)) click.echo("\nSchema before:\n") click.echo(textwrap.indent(prev_schema, " ") or " (empty)") click.echo() @@ -3585,7 +3594,9 @@ def migrate(db_path, migrations, stop_before, list_, verbose): names = {m.name for m in migration_set.pending(db)} names.update(m.name for m in migration_set.applied(db)) known_names.update(names) - known_names.update(f"{migration_set.name}:{name}" for name in names) + known_names.update( + "{}:{}".format(migration_set.name, name) for name in names + ) unknown = [value for value in stop_before if value not in known_names] if unknown: raise click.ClickException( @@ -3641,7 +3652,7 @@ def _render_common(title, values): return "" lines = [title] for value, count in values: - lines.append(f" {count}: {value}") + lines.append(" {}: {}".format(count, value)) return "\n".join(lines) @@ -3711,7 +3722,7 @@ def maybe_json(value): if not isinstance(value, str): return value stripped = value.strip() - if not (stripped.startswith(("{", "["))): + if not (stripped.startswith("{") or stripped.startswith("[")): return value try: return json.loads(stripped) @@ -3729,7 +3740,7 @@ def json_binary(value): def verify_is_dict(doc): if not isinstance(doc, dict): raise click.ClickException( - f"Rows must all be dictionaries, got: {repr(doc)[:1000]}" + "Rows must all be dictionaries, got: {}".format(repr(doc)[:1000]) ) return doc @@ -3757,14 +3768,14 @@ def _register_functions(db, functions): try: functions = pathlib.Path(functions).read_text() except FileNotFoundError: - raise click.ClickException(f"File not found: {functions}") + raise click.ClickException("File not found: {}".format(functions)) sqlite3.enable_callback_tracebacks(True) globals = {} try: - exec(functions, globals) # noqa: S102 + exec(functions, globals) except SyntaxError as ex: - raise click.ClickException(f"Error in functions definition: {ex}") + raise click.ClickException("Error in functions definition: {}".format(ex)) # Register all callables in the locals dict: for name, value in globals.items(): if callable(value) and not name.startswith("_"): @@ -3785,12 +3796,12 @@ def _rows_from_code(code): try: code = pathlib.Path(code).read_text() except FileNotFoundError: - raise click.ClickException(f"File not found: {code}") + raise click.ClickException("File not found: {}".format(code)) namespace = {} try: - exec(code, namespace) # noqa: S102 + exec(code, namespace) except SyntaxError as ex: - raise click.ClickException(f"Error in --code: {ex}") + raise click.ClickException("Error in --code: {}".format(ex)) rows = namespace.get("rows") if callable(rows): rows = rows() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 713b110..d709fb9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,4 +1,19 @@ +from .utils import ( + chunks, + dedupe_keys, + hash_record, + sqlite3, + OperationalError, + suggest_column_types, + types_for_column_types, + column_affinity, + progressbar, + find_spatialite, +) import binascii +from collections import namedtuple +from dataclasses import dataclass, field +from collections.abc import Mapping import contextlib import datetime import decimal @@ -10,35 +25,25 @@ import os import pathlib import re import secrets -import textwrap -import uuid -from collections import namedtuple -from collections.abc import Callable, Generator, Iterable, Mapping, Sequence -from dataclasses import dataclass, field -from types import TracebackType -from typing import ( - Any, - Union, - cast, -) - from sqlite_fts4 import rank_bm25 -from typing_extensions import Self - -from sqlite_utils.plugins import ensure_plugins_loaded, pm - -from .utils import ( - OperationalError, - chunks, - column_affinity, - dedupe_keys, - find_spatialite, - hash_record, - progressbar, - sqlite3, - suggest_column_types, - types_for_column_types, +import textwrap +from typing import ( + cast, + Any, + Callable, + Dict, + Generator, + Iterable, + Sequence, + Set, + Type, + Union, + Optional, + List, + Tuple, ) +import uuid +from sqlite_utils.plugins import ensure_plugins_loaded, pm try: iterdump = importlib.import_module("sqlite_dump").iterdump @@ -221,11 +226,11 @@ class ForeignKey: table: str # column/other_column are None for compound keys, which would break # ordering against str values - comparison uses columns/other_columns - column: str | None = field(compare=False) + column: Optional[str] = field(compare=False) other_table: str - other_column: str | None = field(compare=False) - columns: tuple[str, ...] = () - other_columns: tuple[str, ...] = () + other_column: Optional[str] = field(compare=False) + columns: Tuple[str, ...] = () + other_columns: Tuple[str, ...] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" @@ -254,9 +259,9 @@ def _fk_actions_sql(fk: ForeignKey) -> str: "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." actions = "" if fk.on_update and fk.on_update != "NO ACTION": - actions += f" ON UPDATE {fk.on_update}" + actions += " ON UPDATE {}".format(fk.on_update) if fk.on_delete and fk.on_delete != "NO ACTION": - actions += f" ON DELETE {fk.on_delete}" + actions += " ON DELETE {}".format(fk.on_delete) return actions @@ -273,20 +278,20 @@ class TransformError(Exception): # A single column name, or a tuple of columns for a compound foreign key -ForeignKeyColumns = str | tuple[str, ...] | list[str] +ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] # (table, column(s), other_table, other_column(s)) -ForeignKeyTuple = tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] +ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] -ForeignKeyIndicator = ( - str - | ForeignKey - | tuple[ForeignKeyColumns, str] - | tuple[ForeignKeyColumns, str, ForeignKeyColumns] - | ForeignKeyTuple -) +ForeignKeyIndicator = Union[ + str, + ForeignKey, + Tuple[ForeignKeyColumns, str], + Tuple[ForeignKeyColumns, str, ForeignKeyColumns], + ForeignKeyTuple, +] -ForeignKeysType = Iterable[ForeignKeyIndicator] | list[ForeignKeyIndicator] +ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] class Default: @@ -295,7 +300,7 @@ class Default: DEFAULT = Default() -Tracer = Callable[[str, Sequence[Any] | dict[str, Any] | None], None] +Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: @@ -311,7 +316,7 @@ def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: yield statement_sql -COLUMN_TYPE_MAPPING: dict[Any, str] = { +COLUMN_TYPE_MAPPING: Dict[Any, str] = { float: "REAL", int: "INTEGER", bool: "INTEGER", @@ -507,12 +512,12 @@ class Database: def __init__( self, - filename_or_conn: str | pathlib.Path | sqlite3.Connection | None = None, + filename_or_conn: Optional[Union[str, pathlib.Path, sqlite3.Connection]] = None, memory: bool = False, - memory_name: str | None = None, + memory_name: Optional[str] = None, recreate: bool = False, recursive_triggers: bool = True, - tracer: Tracer | None = None, + tracer: Optional[Tracer] = None, use_counts_table: bool = False, execute_plugins: bool = True, use_old_upsert: bool = False, @@ -527,7 +532,7 @@ class Database: ): raise ValueError("Either specify a filename_or_conn or pass memory=True") if memory_name: - uri = f"file:{memory_name}?mode=memory&cache=shared" + uri = "file:{}?mode=memory&cache=shared".format(memory_name) self.conn = sqlite3.connect( uri, uri=True, @@ -564,7 +569,7 @@ class Database: "transaction handling - connections created with " "autocommit=True or autocommit=False are not supported" ) - self._tracer: Tracer | None = tracer + self._tracer: Optional[Tracer] = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") self._registered_functions: set = set() @@ -574,14 +579,14 @@ class Database: pm.hook.prepare_connection(conn=self.conn) self.strict = strict - def __enter__(self) -> Self: + def __enter__(self) -> "Database": return self def __exit__( self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[object], ) -> None: self.close() @@ -597,8 +602,8 @@ class Database: Nested blocks use SQLite savepoints. """ if self.conn.in_transaction: - savepoint = f"sqlite_utils_{secrets.token_hex(16)}" - self.conn.execute(f"SAVEPOINT {savepoint};") + savepoint = "sqlite_utils_{}".format(secrets.token_hex(16)) + self.conn.execute("SAVEPOINT {};".format(savepoint)) try: yield self except BaseException: @@ -607,11 +612,11 @@ class Database: # anyway would mask the original exception with # "no such savepoint" if self.conn.in_transaction: - self.conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint};") - self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) raise else: - self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) else: self.conn.execute("BEGIN") try: @@ -690,7 +695,9 @@ class Database: self.conn.isolation_level = old_isolation_level @contextlib.contextmanager - def tracer(self, tracer: Tracer | None = None) -> Generator["Database", None, None]: + def tracer( + self, tracer: Optional[Tracer] = None + ) -> Generator["Database", None, None]: """ Context manager to temporarily set a tracer function - all executed SQL queries will be passed to this. @@ -727,15 +734,15 @@ class Database: return self.table(table_name) def __repr__(self) -> str: - return f"" + return "".format(self.conn) def register_function( self, - fn: Callable | None = None, + fn: Optional[Callable] = None, deterministic: bool = False, replace: bool = False, - name: str | None = None, - ) -> Callable[[Callable], Callable] | None: + name: Optional[str] = None, + ) -> Optional[Callable[[Callable], Callable]]: """ ``fn`` will be made available as a function within SQL, with the same name and number of arguments. Can be used as a decorator:: @@ -763,7 +770,7 @@ class Database: arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: return fn - kwargs: dict[str, bool] = {} + kwargs: Dict[str, bool] = {} registered = False if deterministic: # Try this, but fall back if sqlite3.NotSupportedError @@ -789,7 +796,7 @@ class Database: "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True, replace=True) - def attach(self, alias: str, filepath: str | pathlib.Path) -> None: + def attach(self, alias: str, filepath: Union[str, pathlib.Path]) -> None: """ Attach another SQLite database file to this connection with the specified alias, equivalent to:: @@ -798,13 +805,15 @@ class Database: :param alias: Alias name to use :param filepath: Path to SQLite database file on disk """ - attach_sql = f""" - ATTACH DATABASE '{pathlib.Path(filepath).resolve()!s}' AS {quote_identifier(alias)}; - """.strip() + attach_sql = """ + ATTACH DATABASE '{}' AS {}; + """.format( + str(pathlib.Path(filepath).resolve()), quote_identifier(alias) + ).strip() self.execute(attach_sql) def query( - self, sql: str, params: Sequence | dict[str, Any] | None = None + self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None ) -> Generator[dict, None, None]: """ Execute ``sql`` and return an iterable of dictionaries representing each row. @@ -882,7 +891,7 @@ class Database: self.conn.execute('RELEASE "sqlite_utils_query"') def execute( - self, sql: str, parameters: Sequence | dict[str, Any] | None = None + self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None ) -> sqlite3.Cursor: """ Execute SQL query and return a ``sqlite3.Cursor``. @@ -951,7 +960,7 @@ class Database: :param table_name: Name of the table """ if table_name in self.view_names(): - raise NoTable(f"Table {table_name} is actually a view") + raise NoTable("Table {} is actually a view".format(table_name)) kwargs.setdefault("strict", self.strict) return Table(self, table_name, **kwargs) @@ -964,9 +973,11 @@ class Database: if view_name not in self.view_names(): if view_name in self.table_names(): raise NoView( - f"View {view_name} does not exist - {view_name} is a table" + "View {name} does not exist - {name} is a table".format( + name=view_name + ) ) - raise NoView(f"View {view_name} does not exist") + raise NoView("View {} does not exist".format(view_name)) return View(self, view_name) def quote(self, value: str) -> str: @@ -1002,7 +1013,9 @@ class Database: query += '"' bits = _quote_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) def quote_default_value(self, value: str) -> str: if any( @@ -1023,11 +1036,11 @@ class Database: if str(value).endswith(")"): # Expr - return f"({value})" + return "({})".format(value) return self.quote(value) - def table_names(self, fts4: bool = False, fts5: bool = False) -> list[str]: + def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: """ List of string table names in this database. @@ -1042,7 +1055,7 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self) -> list[str]: + def view_names(self) -> List[str]: "List of string view names in this database." return [ r[0] @@ -1052,17 +1065,17 @@ class Database: ] @property - def tables(self) -> list["Table"]: + def tables(self) -> List["Table"]: "List of Table objects in this database." return [self.table(name) for name in self.table_names()] @property - def views(self) -> list["View"]: + def views(self) -> List["View"]: "List of View objects in this database." return [self.view(name) for name in self.view_names()] @property - def triggers(self) -> list[Trigger]: + def triggers(self) -> List[Trigger]: "List of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) @@ -1072,7 +1085,7 @@ class Database: ] @property - def triggers_dict(self) -> dict[str, str]: + def triggers_dict(self) -> Dict[str, str]: "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @@ -1094,12 +1107,14 @@ class Database: "Does this database support STRICT mode?" if not hasattr(self, "_supports_strict"): try: - table_name = f"t{secrets.token_hex(16)}" + table_name = "t{}".format(secrets.token_hex(16)) with self.atomic(): - self.conn.execute(f"create table {table_name} (name text) strict") - self.conn.execute(f"drop table {table_name}") + self.conn.execute( + "create table {} (name text) strict".format(table_name) + ) + self.conn.execute("drop table {}".format(table_name)) self._supports_strict = True - except sqlite3.OperationalError: + except Exception: self._supports_strict = False return self._supports_strict @@ -1107,28 +1122,32 @@ class Database: def supports_on_conflict(self) -> bool: # SQLite's upsert is implemented as INSERT INTO ... ON CONFLICT DO ... if not hasattr(self, "_supports_on_conflict"): - table_name = f"t{secrets.token_hex(16)}" + table_name = "t{}".format(secrets.token_hex(16)) try: with self.atomic(): self.conn.execute( - f"create table {table_name} (id integer primary key, name text)" + "create table {} (id integer primary key, name text)".format( + table_name + ) ) self.conn.execute( - f"insert into {table_name} (id, name) values (1, 'one')" + "insert into {} (id, name) values (1, 'one')".format(table_name) ) self.conn.execute( - f"insert into {table_name} (id, name) values (1, 'two') " - "on conflict do update set name = 'two'" + ( + "insert into {} (id, name) values (1, 'two') " + "on conflict do update set name = 'two'" + ).format(table_name) ) self._supports_on_conflict = True - except sqlite3.OperationalError: + except Exception: self._supports_on_conflict = False finally: - self.conn.execute(f"drop table if exists {table_name}") + self.conn.execute("drop table if exists {}".format(table_name)) return self._supports_on_conflict @property - def sqlite_version(self) -> tuple[int, ...]: + def sqlite_version(self) -> Tuple[int, ...]: "Version of SQLite, as a tuple of integers for example ``(3, 36, 0)``." row = self.execute("select sqlite_version()").fetchall()[0] return tuple(map(int, row[0].split("."))) @@ -1172,7 +1191,7 @@ class Database: # guarantee of atomic() and of user-managed transactions if self.conn.in_transaction: raise TransactionError( - f"{operation} cannot be used while a transaction is open" + "{} cannot be used while a transaction is open".format(operation) ) def _ensure_counts_table(self) -> None: @@ -1193,14 +1212,14 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables: Iterable[str] | None = None) -> dict[str, int]: + def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: """ Return ``{table_name: count}`` dictionary of cached counts for specified tables, or all tables if ``tables`` not provided. :param tables: Subset list of tables to return counts for. """ - sql = f'select "table", count from {self._counts_table_name}' + sql = 'select "table", count from {}'.format(self._counts_table_name) tables_list = list(tables) if tables else None if tables_list: sql += ' where "table" in ({})'.format(", ".join("?" for _ in tables_list)) @@ -1222,13 +1241,13 @@ class Database: ) def execute_returning_dicts( - self, sql: str, params: Sequence | dict[str, Any] | None = None - ) -> list[dict]: + self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None + ) -> List[dict]: return list(self.query(sql, params)) def resolve_foreign_keys( self, name: str, foreign_keys: ForeignKeysType - ) -> list[ForeignKey]: + ) -> List[ForeignKey]: """ Given a list of differing foreign_keys definitions, return a list of fully resolved ForeignKey() named tuples. @@ -1255,7 +1274,7 @@ class Database: fks.append(ForeignKey(name, fk, other_table, other_column)) continue if not isinstance(fk, (tuple, list)): - raise ValueError( # noqa: TRY004 + raise ValueError( "foreign_keys= should be a list of tuples, " "ForeignKey objects or column name strings" ) @@ -1263,7 +1282,9 @@ class Database: if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( - f"First item in {tuple_or_list} should have been {name}" + "First item in {} should have been {}".format( + tuple_or_list, name + ) ) tuple_or_list = tuple_or_list[1:] if len(tuple_or_list) not in (2, 3): @@ -1278,8 +1299,8 @@ class Database: if len(tuple_or_list) == 3: if not isinstance(tuple_or_list[2], (list, tuple)): raise ValueError( - f"Compound foreign key {tuple(tuple_or_list)} should reference a tuple " - "of other columns" + "Compound foreign key {} should reference a tuple " + "of other columns".format(tuple(tuple_or_list)) ) other_columns = tuple(tuple_or_list[2]) else: @@ -1287,8 +1308,8 @@ class Database: other_columns = tuple(self.table(other_table).pks) if len(columns) != len(other_columns): raise ValueError( - f"Compound foreign key {tuple(tuple_or_list)} should have the same number " - "of columns on both sides" + "Compound foreign key {} should have the same number " + "of columns on both sides".format(tuple(tuple_or_list)) ) if len(columns) == 1: # Single-column key passed as a one-item list @@ -1368,15 +1389,15 @@ class Database: def create_table_sql( self, name: str, - columns: dict[str, Any], - pk: Any | None = None, - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, - hash_id: str | None = None, - hash_id_columns: Iterable[str] | None = None, - extracts: dict[str, str] | list[str] | None = None, + columns: Dict[str, Any], + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, strict: bool = False, ) -> str: @@ -1398,7 +1419,7 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - resolved_fks: list[ForeignKey] = [ + resolved_fks: List[ForeignKey] = [ self._resolve_foreign_key_casing(fk, columns) for fk in self.resolve_foreign_keys(name, foreign_keys or []) ] @@ -1428,11 +1449,15 @@ class Database: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): raise ValueError( - f"not_null set {not_null!r} includes items not in columns {set(columns.keys())!r}" + "not_null set {} includes items not in columns {}".format( + repr(not_null), repr(set(columns.keys())) + ) ) if not all(n in columns for n in defaults): raise ValueError( - f"defaults set {set(defaults)!r} includes items not in columns {set(columns.keys())!r}" + "defaults set {} includes items not in columns {}".format( + repr(set(defaults)), repr(set(columns.keys())) + ) ) column_items = list(columns.items()) if column_order is not None: @@ -1452,7 +1477,9 @@ class Database: if other_column != "rowid" and not any( c for c in self[fk.other_table].columns if c.name == other_column ): - raise AlterError(f"No such column: {fk.other_table}.{other_column}") + raise AlterError( + "No such column: {}.{}".format(fk.other_table, other_column) + ) column_defs = [] # ensure pk is a tuple @@ -1473,12 +1500,16 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - f"DEFAULT {self.quote_default_value(defaults[column_name])}" + "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: fk = foreign_keys_by_column[column_name] column_extras.append( - f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" + "REFERENCES {}({}){}".format( + quote_identifier(fk.other_table), + quote_identifier(cast(str, fk.other_column)), + _fk_actions_sql(fk), + ) ) column_type_str = COLUMN_TYPE_MAPPING[column_type] # Special case for strict tables to map FLOAT to REAL @@ -1535,15 +1566,15 @@ class Database: def create_table( self, name: str, - columns: dict[str, Any], - pk: Any | None = None, - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, - hash_id: str | None = None, - hash_id_columns: Iterable[str] | None = None, - extracts: dict[str, str] | list[str] | None = None, + columns: Dict[str, Any], + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, @@ -1587,11 +1618,11 @@ class Database: resolve_casing(col_name, existing_columns): col_type for col_name, col_type in columns.items() } - missing_columns = { - col_name: col_type + missing_columns = dict( + (col_name, col_type) for col_name, col_type in columns.items() if col_name not in existing_columns - } + ) columns_to_drop = [ column for column in existing_columns if column not in columns ] @@ -1678,7 +1709,9 @@ class Database: :param new_name: Name to rename it to """ self.execute( - f"ALTER TABLE {quote_identifier(name)} RENAME TO {quote_identifier(new_name)}" + "ALTER TABLE {} RENAME TO {}".format( + quote_identifier(name), quote_identifier(new_name) + ) ) def create_view( @@ -1694,20 +1727,23 @@ class Database: """ if ignore and replace: raise ValueError("Use one or the other of ignore/replace, not both") - create_sql = f"CREATE VIEW {quote_identifier(name)} AS {sql}" - if (ignore or replace) and name in self.view_names(): - # View exists already - if ignore: - return self - elif replace: - # If SQL is the same, do nothing - if create_sql == self[name].schema: + create_sql = "CREATE VIEW {name} AS {sql}".format( + name=quote_identifier(name), sql=sql + ) + if ignore or replace: + # Does view exist already? + if name in self.view_names(): + if ignore: return self - self[name].drop() + elif replace: + # If SQL is the same, do nothing + if create_sql == self[name].schema: + return self + self[name].drop() self.execute(create_sql) return self - def m2m_table_candidates(self, table: str, other_table: str) -> list[str]: + def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: """ Given two table names returns the name of tables that could define a many-to-many relationship between those two tables, based on having @@ -1726,7 +1762,7 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[ForeignKey | ForeignKeyTuple] + self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. @@ -1746,7 +1782,7 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create: list[ForeignKey] = [] + foreign_keys_to_create: List[ForeignKey] = [] # Verify that all tables and columns exist for fk in foreign_keys: @@ -1787,7 +1823,7 @@ class Database: table = fk_object.table other_table = fk_object.other_table if not self.table(table).exists(): - raise AlterError(f"No such table: {table}") + raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) fk_object = self._resolve_foreign_key_casing( fk_object, table_obj.columns_dict @@ -1796,16 +1832,18 @@ class Database: other_columns = fk_object.other_columns for column in columns: if column not in table_obj.columns_dict: - raise AlterError(f"No such column: {column} in {table}") + raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): - raise AlterError(f"No such other_table: {other_table}") + raise AlterError("No such other_table: {}".format(other_table)) for other_column in other_columns: if ( other_column != "rowid" and other_column not in self[other_table].columns_dict ): raise AlterError( - f"No such other_column: {other_column} in {other_table}" + "No such other_column: {} in {}".format( + other_column, other_table + ) ) # Silently skip foreign keys that exist already - but only if # they match exactly, including ON DELETE/ON UPDATE actions @@ -1836,7 +1874,7 @@ class Database: ) # Group them by table - by_table: dict[str, list[ForeignKey]] = {} + by_table: Dict[str, List[ForeignKey]] = {} for fk_object in foreign_keys_to_create: by_table.setdefault(fk_object.table, []).append(fk_object) @@ -1861,7 +1899,7 @@ class Database: "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") - def analyze(self, name: str | None = None) -> None: + def analyze(self, name: Optional[str] = None) -> None: """ Run ``ANALYZE`` against the entire database or a named table or index. @@ -1869,7 +1907,7 @@ class Database: """ sql = "ANALYZE" if name is not None: - sql += f" {quote_identifier(name)}" + sql += " {}".format(quote_identifier(name)) self.execute(sql) def iterdump(self) -> Generator[str, None, None]: @@ -1884,7 +1922,7 @@ class Database: "conn.iterdump() not found - try pip install sqlite-dump" ) - def init_spatialite(self, path: str | None = None) -> bool: + def init_spatialite(self, path: Optional[str] = None) -> bool: """ The ``init_spatialite`` method will load and initialize the SpatiaLite extension. The ``path`` argument should be an absolute path to the compiled extension, which @@ -1942,8 +1980,8 @@ class Queryable: def count_where( self, - where: str | None = None, - where_args: Sequence | dict[str, Any] | None = None, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, ) -> int: """ Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count. @@ -1952,7 +1990,7 @@ class Queryable: :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` parameters, or a dictionary for ``id > :id`` """ - sql = f"select count(*) from {quote_identifier(self.name)}" + sql = "select count(*) from {}".format(quote_identifier(self.name)) if where is not None: sql += " where " + where return self.db.execute(sql, where_args or []).fetchone()[0] @@ -1967,19 +2005,19 @@ class Queryable: return self.count_where() @property - def rows(self) -> Generator[dict[str, Any], None, None]: + def rows(self) -> Generator[Dict[str, Any], None, None]: "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where: str | None = None, - where_args: Sequence | dict[str, Any] | None = None, - order_by: str | None = None, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + order_by: Optional[str] = None, select: str = "*", - limit: int | None = None, - offset: int | None = None, - ) -> Generator[dict[str, Any], None, None]: + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[Dict[str, Any], None, None]: """ Iterate over every row in this table or view that matches the specified where clause. @@ -1995,15 +2033,15 @@ class Queryable: """ if not self.exists(): return - sql = f"select {select} from {quote_identifier(self.name)}" + sql = "select {} from {}".format(select, quote_identifier(self.name)) if where is not None: sql += " where " + where if order_by is not None: sql += " order by " + order_by if limit is not None: - sql += f" limit {limit}" + sql += " limit {}".format(limit) if offset is not None: - sql += f" offset {offset}" + sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: @@ -2011,12 +2049,12 @@ class Queryable: def pks_and_rows_where( self, - where: str | None = None, - where_args: Sequence | dict[str, Any] | None = None, - order_by: str | None = None, - limit: int | None = None, - offset: int | None = None, - ) -> Generator[tuple[Any, dict[str, Any]], None, None]: + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Generator[Tuple[Any, Dict[str, Any]], None, None]: """ Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. @@ -2058,17 +2096,17 @@ class Queryable: yield row_pk, row @property - def columns(self) -> list["Column"]: + def columns(self) -> List["Column"]: "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute( - f"PRAGMA table_info({quote_identifier(self.name)})" + "PRAGMA table_info({})".format(quote_identifier(self.name)) ).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self) -> dict[str, Any]: + def columns_dict(self) -> Dict[str, Any]: "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @@ -2108,48 +2146,48 @@ class Table(Queryable): """ #: The ``rowid`` of the last inserted, updated or selected row. - last_rowid: int | None = None + last_rowid: Optional[int] = None #: The primary key of the last inserted, updated or selected row. - last_pk: Any | None = None + last_pk: Optional[Any] = None def __init__( self, db: Database, name: str, - pk: Any | None = None, - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, + pk: Optional[Any] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, batch_size: int = 100, - hash_id: str | None = None, - hash_id_columns: Iterable[str] | None = None, + hash_id: Optional[str] = None, + hash_id_columns: Optional[Iterable[str]] = None, alter: bool = False, ignore: bool = False, replace: bool = False, - extracts: dict[str, str] | list[str] | None = None, - conversions: dict | None = None, - columns: dict[str, Any] | None = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[dict] = None, + columns: Optional[Dict[str, Any]] = None, strict: bool = False, ): super().__init__(db, name) - self._defaults = { - "pk": pk, - "foreign_keys": foreign_keys, - "column_order": column_order, - "not_null": not_null, - "defaults": defaults, - "batch_size": batch_size, - "hash_id": hash_id, - "hash_id_columns": hash_id_columns, - "alter": alter, - "ignore": ignore, - "replace": replace, - "extracts": extracts, - "conversions": conversions or {}, - "columns": columns, - "strict": strict, - } + self._defaults = dict( + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + batch_size=batch_size, + hash_id=hash_id, + hash_id_columns=hash_id_columns, + alter=alter, + ignore=ignore, + replace=replace, + extracts=extracts, + conversions=conversions or {}, + columns=columns, + strict=strict, + ) def __repr__(self) -> str: return "".format( @@ -2174,7 +2212,7 @@ class Table(Queryable): return self.name in self.db.table_names() @property - def pks(self) -> list[str]: + def pks(self) -> List[str]: """ Primary key columns for this table, in PRIMARY KEY declaration order - ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each @@ -2196,7 +2234,7 @@ class Table(Queryable): "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values: list | tuple | str | int) -> dict: + def get(self, pk_values: Union[list, tuple, str, int]) -> dict: """ Return row (as dictionary) for the specified primary key. @@ -2215,17 +2253,17 @@ class Table(Queryable): ) ) - wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] rows = self.rows_where(" and ".join(wheres), pk_values) try: - row = next(iter(rows)) + row = list(rows)[0] self.last_pk = last_pk return row - except StopIteration: + except IndexError: raise NotFoundError @property - def foreign_keys(self) -> list["ForeignKey"]: + def foreign_keys(self) -> List["ForeignKey"]: """ List of foreign keys defined on this table. @@ -2235,12 +2273,12 @@ class Table(Queryable): """ # PRAGMA foreign_key_list returns one row per column, grouped by "id" # with "seq" giving the column order within a compound foreign key. - by_id: dict[int, list] = {} + by_id: Dict[int, list] = {} for row in self.db.execute( - f"PRAGMA foreign_key_list({quote_identifier(self.name)})" + "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) ).fetchall(): if row is not None: - id, seq, table_name, from_, to_, on_update, on_delete, _match = row + id, seq, table_name, from_, to_, on_update, on_delete, match = row by_id.setdefault(id, []).append( (seq, table_name, from_, to_, on_update, on_delete) ) @@ -2273,7 +2311,7 @@ class Table(Queryable): return fks @property - def virtual_table_using(self) -> str | None: + def virtual_table_using(self) -> Optional[str]: "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: @@ -2281,16 +2319,18 @@ class Table(Queryable): return match.groupdict()["using"].upper() @property - def indexes(self) -> list[Index]: + def indexes(self) -> List[Index]: "List of indexes defined on this table." - sql = f'PRAGMA index_list("{self.name}")' + sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name + '"{}"'.format(index_name) + if not index_name.startswith('"') + else index_name ) - column_sql = f"PRAGMA index_info({index_name_quoted})" + column_sql = "PRAGMA index_info({})".format(index_name_quoted) columns = [] for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) @@ -2303,16 +2343,18 @@ class Table(Queryable): return indexes @property - def xindexes(self) -> list[XIndex]: + def xindexes(self) -> List[XIndex]: "List of indexes defined on this table using the more detailed ``XIndex`` format." - sql = f'PRAGMA index_list("{self.name}")' + sql = 'PRAGMA index_list("{}")'.format(self.name) indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name + '"{}"'.format(index_name) + if not index_name.startswith('"') + else index_name ) - column_sql = f"PRAGMA index_xinfo({index_name_quoted})" + column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) index_columns = [] for info in self.db.execute(column_sql).fetchall(): index_columns.append(XIndexColumn(*info)) @@ -2320,7 +2362,7 @@ class Table(Queryable): return indexes @property - def triggers(self) -> list[Trigger]: + def triggers(self) -> List[Trigger]: "List of triggers defined on this table." return [ Trigger(*r) @@ -2332,12 +2374,12 @@ class Table(Queryable): ] @property - def triggers_dict(self) -> dict[str, str]: + def triggers_dict(self) -> Dict[str, str]: "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def default_values(self) -> dict[str, Any]: + def default_values(self) -> Dict[str, Any]: "``{column_name: default_value}`` dictionary of default values for columns in this table." return { column.name: _decode_default_value(column.default_value) @@ -2354,20 +2396,20 @@ class Table(Queryable): def create( self, - columns: dict[str, Any], - pk: Any | None = DEFAULT, - foreign_keys: ForeignKeysType | None | Default = DEFAULT, - column_order: list[str] | None | Default = DEFAULT, - not_null: Iterable[str] | None | Default = DEFAULT, - defaults: dict[str, Any] | None | Default = DEFAULT, - hash_id: str | None | Default = DEFAULT, - hash_id_columns: Iterable[str] | None | Default = DEFAULT, - extracts: dict[str, str] | list[str] | None | Default = DEFAULT, + columns: Dict[str, Any], + pk: Optional[Any] = DEFAULT, + foreign_keys: Union[Optional[ForeignKeysType], Default] = DEFAULT, + column_order: Union[Optional[List[str]], Default] = DEFAULT, + not_null: Union[Optional[Iterable[str]], Default] = DEFAULT, + defaults: Union[Optional[Dict[str, Any]], Default] = DEFAULT, + hash_id: Union[Optional[str], Default] = DEFAULT, + hash_id_columns: Union[Optional[Iterable[str]], Default] = DEFAULT, + extracts: Union[Optional[Union[Dict[str, str], List[str]]], Default] = DEFAULT, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, transform: bool = False, - strict: bool | Default = DEFAULT, + strict: Union[bool, Default] = DEFAULT, ) -> "Table": """ Create a table with the specified columns. @@ -2451,25 +2493,28 @@ class Table(Queryable): if not self.exists(): raise NoTable(f"Table {self.name} does not exist") with self.db.atomic(): - sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" + sql = "CREATE TABLE {} AS SELECT * FROM {};".format( + quote_identifier(new_name), + quote_identifier(self.name), + ) self.db.execute(sql) return self.db.table(new_name) def transform( self, *, - types: dict | None = None, - rename: dict | None = None, - drop: Iterable | None = None, - pk: Any | None = DEFAULT, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, - drop_foreign_keys: Iterable[str] | None = None, - add_foreign_keys: ForeignKeysType | None = None, - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - keep_table: str | None = None, - strict: bool | None = None, + types: Optional[dict] = None, + rename: Optional[dict] = None, + drop: Optional[Iterable] = None, + pk: Optional[Any] = DEFAULT, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + drop_foreign_keys: Optional[Iterable[str]] = None, + add_foreign_keys: Optional[ForeignKeysType] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + keep_table: Optional[str] = None, + strict: Optional[bool] = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2477,11 +2522,6 @@ class Table(Queryable): See :ref:`python_api_transform` for full details. - Raises :py:class:`sqlite_utils.db.TransactionError` if called while a - transaction is open with ``PRAGMA foreign_keys`` enabled and the table - is referenced by foreign keys with destructive ``ON DELETE`` actions - - see :ref:`python_api_transform_foreign_keys_transactions`. - :param types: Columns that should have their type changed, for example ``{"weight": float}`` :param rename: Columns to rename, for example ``{"headline": "title"}`` :param drop: Columns to drop @@ -2526,36 +2566,6 @@ class Table(Queryable): should_defer_foreign_keys = ( pragma_foreign_keys_was_on and already_in_transaction ) - if should_defer_foreign_keys: - # PRAGMA foreign_keys is a no-op inside a transaction, and - # defer_foreign_keys only defers violation checks, not ON DELETE - # actions - so dropping the old table would still fire destructive - # actions on any tables that reference it. Refuse rather than - # silently modify or delete those rows. - destructive_fks = [ - (table.name, fk) - for table in self.db.tables - for fk in table.foreign_keys - if fk.other_table == self.name - and fk.on_delete in ("CASCADE", "SET NULL", "SET DEFAULT") - ] - if destructive_fks: - raise TransactionError( - "Cannot transform table {table} while a transaction is open: " - "PRAGMA foreign_keys cannot be changed inside a transaction, " - "and the table is referenced by foreign keys with ON DELETE " - "actions that would fire when the old table is dropped: " - "{fks}. Call transform() outside of the transaction, or " - 'execute "PRAGMA foreign_keys = off" before opening it.'.format( - table=self.name, - fks=", ".join( - "{}.{} (ON DELETE {})".format( - table_name, ", ".join(fk.columns), fk.on_delete - ) - for table_name, fk in destructive_fks - ), - ) - ) defer_foreign_keys_was_on = False try: if should_disable_foreign_keys: @@ -2588,20 +2598,20 @@ class Table(Queryable): def transform_sql( self, *, - types: dict | None = None, - rename: dict | None = None, - drop: Iterable | None = None, - pk: Any | None = DEFAULT, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, - drop_foreign_keys: Iterable | None = None, - add_foreign_keys: ForeignKeysType | None = None, - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - tmp_suffix: str | None = None, - keep_table: str | None = None, - strict: bool | None = None, - ) -> list[str]: + types: Optional[dict] = None, + rename: Optional[dict] = None, + drop: Optional[Iterable] = None, + pk: Optional[Any] = DEFAULT, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + drop_foreign_keys: Optional[Iterable] = None, + add_foreign_keys: Optional[ForeignKeysType] = None, + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + tmp_suffix: Optional[str] = None, + keep_table: Optional[str] = None, + strict: Optional[bool] = None, + ) -> List[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2644,7 +2654,7 @@ class Table(Queryable): if isinstance(not_null, dict): not_null = { resolve_casing(c, existing_columns): v - for c, v in cast(dict[str, Any], not_null).items() + for c, v in cast(Dict[str, Any], not_null).items() } elif isinstance(not_null, set): not_null = {resolve_casing(c, existing_columns) for c in not_null} @@ -2655,7 +2665,7 @@ class Table(Queryable): if column_order is not None: column_order = [resolve_casing(c, existing_columns) for c in column_order] - create_table_foreign_keys: list[ForeignKeyIndicator] = [] + create_table_foreign_keys: List[ForeignKeyIndicator] = [] if foreign_keys is not None: if add_foreign_keys is not None: @@ -2732,7 +2742,9 @@ class Table(Queryable): for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): create_table_foreign_keys.append(fk_with_renamed_columns(fk)) - new_table_name = f"{self.name}_new_{tmp_suffix or os.urandom(6).hex()}" + new_table_name = "{}_new_{}".format( + self.name, tmp_suffix or os.urandom(6).hex() + ) current_column_pairs = list(self.columns_dict.items()) new_column_pairs = [] copy_from_to = {column: column for column, _ in current_column_pairs} @@ -2777,7 +2789,9 @@ class Table(Queryable): pass else: raise ValueError( - f"not_null must be a dict or a set or None, it was {not_null!r}" + "not_null must be a dict or a set or None, it was {}".format( + repr(not_null) + ) ) # defaults= create_table_defaults = { @@ -2827,13 +2841,17 @@ class Table(Queryable): # Drop (or keep) the old table if keep_table: sqls.append( - f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" + "ALTER TABLE {} RENAME TO {};".format( + quote_identifier(self.name), quote_identifier(keep_table) + ) ) else: - sqls.append(f"DROP TABLE {quote_identifier(self.name)};") + sqls.append("DROP TABLE {};".format(quote_identifier(self.name))) # Rename the new one sqls.append( - f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" + "ALTER TABLE {} RENAME TO {};".format( + quote_identifier(new_table_name), quote_identifier(self.name) + ) ) # Re-add existing indexes for index in self.indexes: @@ -2851,7 +2869,7 @@ class Table(Queryable): if keep_table: sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") for col in index.columns: - if col in rename or col in drop: + if col in rename.keys() or col in drop: raise TransformError( f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " f"You must manually drop this index prior to running this transformation " @@ -2863,10 +2881,10 @@ class Table(Queryable): def extract( self, - columns: str | Iterable[str], - table: str | None = None, - fk_column: str | None = None, - rename: dict[str, str] | None = None, + columns: Union[str, Iterable[str]], + table: Optional[str] = None, + fk_column: Optional[str] = None, + rename: Optional[Dict[str, str]] = None, ) -> "Table": """ Extract specified columns into a separate table. @@ -2885,13 +2903,15 @@ class Table(Queryable): rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( - f"Invalid columns {columns} for table with columns {list(self.columns_dict.keys())}" + "Invalid columns {} for table with columns {}".format( + columns, list(self.columns_dict.keys()) + ) ) with self.db.atomic(): table = table or "_".join(columns) lookup_table = self.db.table(table) - fk_column = fk_column or f"{table}_id" - magic_lookup_column = f"{fk_column}_{os.urandom(6).hex()}" + fk_column = fk_column or "{}_id".format(table) + magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) # Populate the lookup table with all of the extracted unique values lookup_columns_definition = { @@ -2904,12 +2924,16 @@ class Table(Queryable): lookup_table.columns_dict.items() ): raise InvalidColumns( - f"Lookup table {table} already exists but does not have columns {lookup_columns_definition}" + "Lookup table {} already exists but does not have columns {}".format( + table, lookup_columns_definition + ) ) else: lookup_table.create( { - "id": int, + **{ + "id": int, + }, **lookup_columns_definition, }, pk="id", @@ -2919,14 +2943,19 @@ class Table(Queryable): # Rows where every extracted column is null are left alone - they # get a null foreign key and no lookup table record, see #186 all_columns_are_null = " AND ".join( - f"{quote_identifier(c)} IS NULL" for c in columns + "{} IS NULL".format(quote_identifier(c)) for c in columns ) # INSERT OR IGNORE dedupes against the unique index, but unique # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS # comparison so NULL-containing rows match existing lookup rows # instead of being inserted again already_in_lookup = " AND ".join( - f"{quote_identifier(table)}.{quote_identifier(rename.get(column) or column)} IS {quote_identifier(self.name)}.{quote_identifier(column)}" + "{lookup}.{lookup_col} IS {source}.{source_col}".format( + lookup=quote_identifier(table), + lookup_col=quote_identifier(rename.get(column) or column), + source=quote_identifier(self.name), + source_col=quote_identifier(column), + ) for column in columns ) self.db.execute( @@ -2954,10 +2983,12 @@ class Table(Queryable): quote_identifier(magic_lookup_column), quote_identifier(table), where=" AND ".join( - f"{quote_identifier(self.name)}." - f"{quote_identifier(column)} IS " - f"{quote_identifier(table)}." - f"{quote_identifier(rename.get(column) or column)}" + "{}.{} IS {}.{}".format( + quote_identifier(self.name), + quote_identifier(column), + quote_identifier(table), + quote_identifier(rename.get(column) or column), + ) for column in columns ), all_null=all_columns_are_null, @@ -2986,8 +3017,8 @@ class Table(Queryable): def create_index( self, - columns: Iterable[str | DescIndex], - index_name: str | None = None, + columns: Iterable[Union[str, DescIndex]], + index_name: Optional[str] = None, unique: bool = False, if_not_exists: bool = False, find_unique_name: bool = False, @@ -3014,14 +3045,16 @@ class Table(Queryable): columns_sql = [] for column in columns: if isinstance(column, DescIndex): - columns_sql.append(f"{quote_identifier(column)} desc") + columns_sql.append("{} desc".format(quote_identifier(column))) else: columns_sql.append(quote_identifier(column)) suffix = None created_index_name = None while True: - created_index_name = f"{index_name}_{suffix}" if suffix else index_name + created_index_name = ( + "{}_{}".format(index_name, suffix) if suffix else index_name + ) sql = ( textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}{index_name} @@ -3053,7 +3086,7 @@ class Table(Queryable): suffix += 1 continue else: - raise + raise e if analyze: self.db.analyze(created_index_name) return self @@ -3068,17 +3101,19 @@ class Table(Queryable): if index_name not in {index.name for index in self.indexes}: if ignore: return self - raise OperationalError(f"No index named {index_name} on table {self.name}") - self.db.execute(f"DROP INDEX {quote_identifier(index_name)}") + raise OperationalError( + "No index named {} on table {}".format(index_name, self.name) + ) + self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) return self def add_column( self, col_name: str, - col_type: Any | None = None, - fk: str | None = None, - fk_col: str | None = None, - not_null_default: Any | None = None, + col_type: Optional[Any] = None, + fk: Optional[str] = None, + fk_col: Optional[str] = None, + not_null_default: Optional[Any] = None, ): """ Add a column to this table. See :ref:`python_api_add_column`. @@ -3093,12 +3128,12 @@ class Table(Queryable): if fk is not None: # fk must be a valid table if fk not in self.db.table_names(): - raise AlterError(f"table '{fk}' does not exist") + raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: - raise AlterError(f"table '{fk}' has no column {fk_col}") + raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: # automatically set fk_col to first primary_key of fk table pks = sorted( @@ -3115,8 +3150,8 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = ( - f"NOT NULL DEFAULT {self.db.quote_default_value(not_null_default)}" + not_null_sql = "NOT NULL DEFAULT {}".format( + self.db.quote_default_value(not_null_default) ) sql = "ALTER TABLE {} ADD COLUMN {} {col_type}{not_null_default};".format( quote_identifier(self.name), @@ -3136,7 +3171,7 @@ class Table(Queryable): :param ignore: Set to ``True`` to ignore the error if the table does not exist """ try: - self.db.execute(f"DROP TABLE {quote_identifier(self.name)}") + self.db.execute("DROP TABLE {}".format(quote_identifier(self.name))) except sqlite3.OperationalError: if not ignore: raise @@ -3168,14 +3203,16 @@ class Table(Queryable): return existing_tables[table] # If we get here there's no obvious candidate - raise an error raise NoObviousTable( - f"No obvious foreign key table for column '{column}' - tried {possibilities!r}" + "No obvious foreign key table for column '{}' - tried {}".format( + column, repr(possibilities) + ) ) def guess_foreign_column(self, other_table: str) -> str: pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( - f"Could not detect single primary key for table '{other_table}'" + "Could not detect single primary key for table '{}'".format(other_table) ) else: return pks[0].name @@ -3183,8 +3220,8 @@ class Table(Queryable): def add_foreign_key( self, column: ForeignKeyColumns, - other_table: str | None = None, - other_column: ForeignKeyColumns | None = None, + other_table: Optional[str] = None, + other_column: Optional[ForeignKeyColumns] = None, ignore: bool = False, on_delete: str = "NO ACTION", on_update: str = "NO ACTION", @@ -3207,7 +3244,7 @@ class Table(Queryable): # Ensure columns exist for col in columns: if col not in self.columns_dict: - raise AlterError(f"No such column: {col}") + raise AlterError("No such column: {}".format(col)) # If other_table is not specified, attempt to guess it from the column if other_table is None: if len(columns) > 1: @@ -3240,7 +3277,7 @@ class Table(Queryable): not [c for c in self.db[other_table].columns if c.name == other_col] and other_col != "rowid" ): - raise AlterError(f"No such column: {other_table}.{other_col}") + raise AlterError("No such column: {}.{}".format(other_table, other_col)) # Check we do not already have an existing foreign key if any( fk @@ -3341,7 +3378,9 @@ class Table(Queryable): def has_counts_triggers(self) -> bool: "Does this table have triggers setup to update cached counts?" trigger_names = { - f"{self.name}{self.db._counts_table_name}_{suffix}" + "{table}{counts_table}_{suffix}".format( + counts_table=self.db._counts_table_name, table=self.name, suffix=suffix + ) for suffix in ["insert", "delete"] } return trigger_names.issubset(self.triggers_dict.keys()) @@ -3351,7 +3390,7 @@ class Table(Queryable): columns: Iterable[str], fts_version: str = "FTS5", create_triggers: bool = False, - tokenize: str | None = None, + tokenize: Optional[str] = None, replace: bool = False, ): """ @@ -3378,13 +3417,13 @@ class Table(Queryable): table_fts=quote_identifier(self.name + "_fts"), columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, - tokenize=f"\n tokenize='{tokenize}'," if tokenize else "", + tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) ) should_recreate = False - if replace and self.db[f"{self.name}_fts"].exists(): + if replace and self.db["{}_fts".format(self.name)].exists(): # Does the table need to be recreated? - fts_schema = self.db[f"{self.name}_fts"].schema + fts_schema = self.db["{}_fts".format(self.name)].schema if fts_schema != create_fts_sql: should_recreate = True expected_triggers = {self.name + suffix for suffix in ("_ai", "_ad", "_au")} @@ -3403,8 +3442,8 @@ class Table(Queryable): self.populate_fts(columns) if create_triggers: - old_cols = ", ".join(f"old.{quote_identifier(c)}" for c in columns) - new_cols = ", ".join(f"new.{quote_identifier(c)}" for c in columns) + old_cols = ", ".join("old.{}".format(quote_identifier(c)) for c in columns) + new_cols = ", ".join("new.{}".format(quote_identifier(c)) for c in columns) columns_quoted = ", ".join(quote_identifier(c) for c in columns) table = quote_identifier(self.name) table_fts = quote_identifier(self.name + "_fts") @@ -3476,7 +3515,7 @@ class Table(Queryable): with self.db.atomic(): for trigger_name in trigger_names: self.db.execute( - f"DROP TRIGGER IF EXISTS {quote_identifier(trigger_name)}" + "DROP TRIGGER IF EXISTS {}".format(quote_identifier(trigger_name)) ) return self @@ -3494,7 +3533,7 @@ class Table(Queryable): ) return self - def detect_fts(self) -> str | None: + def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" sql = textwrap.dedent(""" SELECT name FROM sqlite_master @@ -3509,8 +3548,8 @@ class Table(Queryable): ) """).strip() args = { - "like": f"%VIRTUAL TABLE%USING FTS%content=[{self.name}]%", - "like2": f'%VIRTUAL TABLE%USING FTS%content="{self.name}"%', + "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), + "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "table": self.name, } rows = self.db.execute(sql, args).fetchall() @@ -3531,11 +3570,11 @@ class Table(Queryable): def search_sql( self, - columns: Iterable[str] | None = None, - order_by: str | None = None, - limit: int | None = None, - offset: int | None = None, - where: str | None = None, + columns: Optional[Iterable[str]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + where: Optional[str] = None, include_rank: bool = False, ) -> str: """ " @@ -3552,16 +3591,16 @@ class Table(Queryable): original = "original_" if self.name == "original" else "original" original_quoted = quote_identifier(original) columns_sql = "*" - columns_with_prefix_sql = f"{original_quoted}.*" + columns_with_prefix_sql = "{}.*".format(original_quoted) if columns: columns_sql = ",\n ".join(quote_identifier(c) for c in columns) columns_with_prefix_sql = ",\n ".join( - f"{original_quoted}.{quote_identifier(c)}" for c in columns + "{}.{}".format(original_quoted, quote_identifier(c)) for c in columns ) fts_table = self.detect_fts() if not fts_table: raise ValueError( - f"Full-text search is not configured for table '{self.name}'" + "Full-text search is not configured for table '{}'".format(self.name) ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using @@ -3584,20 +3623,22 @@ class Table(Queryable): {limit_offset} """).strip() if virtual_table_using == "FTS5": - rank_implementation = f"{fts_table_quoted}.rank" + rank_implementation = "{}.rank".format(fts_table_quoted) else: self.db.register_fts4_bm25() - rank_implementation = f"rank_bm25(matchinfo({fts_table_quoted}, 'pcnalx'))" + rank_implementation = "rank_bm25(matchinfo({}, 'pcnalx'))".format( + fts_table_quoted + ) if include_rank: columns_with_prefix_sql += ",\n " + rank_implementation + " rank" limit_offset = "" if limit is not None: - limit_offset += f" limit {limit}" + limit_offset += " limit {}".format(limit) if offset is not None: - limit_offset += f" offset {offset}" + limit_offset += " offset {}".format(offset) return sql.format( dbtable=quote_identifier(self.name), - where_clause=f"\n where {where}" if where else "", + where_clause="\n where {}".format(where) if where else "", original=original_quoted, columns=columns_sql, columns_with_prefix=columns_with_prefix_sql, @@ -3609,12 +3650,12 @@ class Table(Queryable): def search( self, q: str, - order_by: str | None = None, - columns: Iterable[str] | None = None, - limit: int | None = None, - offset: int | None = None, - where: str | None = None, - where_args: Iterable | dict | None = None, + order_by: Optional[str] = None, + columns: Optional[Iterable[str]] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + where: Optional[str] = None, + where_args: Optional[Union[Iterable, dict]] = None, include_rank: bool = False, quote: bool = False, ) -> Generator[dict, None, None]: @@ -3660,7 +3701,7 @@ class Table(Queryable): def value_or_default(self, key: str, value: Any) -> Any: return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values: list | tuple | str | float) -> "Table": + def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": """ Delete row matching the specified primary key. @@ -3669,7 +3710,7 @@ class Table(Queryable): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) - wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in self.pks] + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in self.pks] sql = "delete from {} where {wheres}".format( quote_identifier(self.name), wheres=" and ".join(wheres) ) @@ -3679,8 +3720,8 @@ class Table(Queryable): def delete_where( self, - where: str | None = None, - where_args: Sequence | dict[str, Any] | None = None, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, analyze: bool = False, ) -> "Table": """ @@ -3695,7 +3736,7 @@ class Table(Queryable): """ if not self.exists(): return self - sql = f"delete from {quote_identifier(self.name)}" + sql = "delete from {}".format(quote_identifier(self.name)) if where is not None: sql += " where " + where with self.db.atomic(): @@ -3706,10 +3747,10 @@ class Table(Queryable): def update( self, - pk_values: list | tuple | str | float, - updates: dict | None = None, + pk_values: Union[list, tuple, str, int, float], + updates: Optional[dict] = None, alter: bool = False, - conversions: dict | None = None, + conversions: Optional[dict] = None, ) -> "Table": """ Execute a SQL ``UPDATE`` against the specified row. @@ -3740,7 +3781,7 @@ class Table(Queryable): "{} = {}".format(quote_identifier(key), conversions.get(key, "?")) ) args.append(jsonify_if_needed(value)) - wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] + wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] args.extend(pk_values) sql = "update {} set {sets} where {wheres}".format( quote_identifier(self.name), @@ -3765,14 +3806,14 @@ class Table(Queryable): def convert( self, - columns: str | list[str], + columns: Union[str, List[str]], fn: Callable, - output: str | None = None, - output_type: Any | None = None, + output: Optional[str] = None, + output_type: Optional[Any] = None, drop: bool = False, multi: bool = False, - where: str | None = None, - where_args: Sequence | dict[str, Any] | None = None, + where: Optional[str] = None, + where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, show_progress: bool = False, ) -> "Table": """ @@ -3829,11 +3870,15 @@ class Table(Queryable): quote_identifier(self.name), sets=", ".join( [ - f"{quote_identifier(output or column)} = {fn_name}({quote_identifier(column)})" + "{} = {}({})".format( + quote_identifier(output or column), + fn_name, + quote_identifier(column), + ) for column in columns ] ), - where=f" where {where}" if where is not None else "", + where=" where {}".format(where) if where is not None else "", ) with self.db.atomic(): self.db.execute(sql, where_args or []) @@ -3846,7 +3891,7 @@ class Table(Queryable): ): # First we execute the function pk_to_values = {} - new_column_types: dict[str, set[type]] = {} + new_column_types: Dict[str, Set[type]] = {} pks = self.pks with progressbar( @@ -3878,17 +3923,15 @@ class Table(Queryable): self.add_column(column_name, column_type) # Run the updates - with ( - progressbar( - length=self.count, silent=not show_progress, label="2: Updating" - ) as bar, - self.db.atomic(), - ): - for pk, updates in pk_to_values.items(): - self.update(pk, updates) - bar.update(1) - if drop: - self.transform(drop=(column,)) + with progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar: + with self.db.atomic(): + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) def build_insert_queries_and_params( self, @@ -4088,7 +4131,9 @@ class Table(Queryable): ) for col in set_cols ), - wheres=" AND ".join(f"{quote_identifier(pk)} = ?" for pk in pks), + wheres=" AND ".join( + "{} = ?".format(quote_identifier(pk)) for pk in pks + ), ) queries_and_params.append( ( @@ -4121,7 +4166,7 @@ class Table(Queryable): replace, ignore, list_mode=False, - ) -> sqlite3.Cursor | None: + ) -> Optional[sqlite3.Cursor]: queries_and_params = self.build_insert_queries_and_params( extracts, chunk, @@ -4191,21 +4236,21 @@ class Table(Queryable): def insert( self, - record: dict[str, Any], + record: Dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order: list[str] | Default | None = DEFAULT, - not_null: Iterable[str] | Default | None = DEFAULT, - defaults: dict[str, Any] | Default | None = DEFAULT, - hash_id: str | Default | None = DEFAULT, - hash_id_columns: Iterable[str] | Default | None = DEFAULT, - alter: bool | Default | None = DEFAULT, - ignore: bool | Default | None = DEFAULT, - replace: bool | Default | None = DEFAULT, - extracts: dict[str, str] | list[str] | Default | None = DEFAULT, - conversions: dict[str, str] | Default | None = DEFAULT, - columns: dict[str, Any] | Default | None = DEFAULT, - strict: bool | Default | None = DEFAULT, + column_order: Optional[Union[List[str], Default]] = DEFAULT, + not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, + defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + hash_id: Optional[Union[str, Default]] = DEFAULT, + hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, + alter: Optional[Union[bool, Default]] = DEFAULT, + ignore: Optional[Union[bool, Default]] = DEFAULT, + replace: Optional[Union[bool, Default]] = DEFAULT, + extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, + conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, + columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, + strict: Optional[Union[bool, Default]] = DEFAULT, ) -> "Table": """ Insert a single record into the table. The table will be created with a schema that matches @@ -4260,7 +4305,10 @@ class Table(Queryable): def insert_all( self, - records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], + records: Union[ + Iterable[Dict[str, Any]], + Iterable[Sequence[Any]], + ], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4357,7 +4405,7 @@ class Table(Queryable): # Detect if we're using list-based iteration or dict-based iteration list_mode = False - column_names: list[str] = [] + column_names: List[str] = [] # Fix up any records with square braces in the column names (only for dict mode) # We'll handle this differently for list mode @@ -4377,7 +4425,7 @@ class Table(Queryable): raise ValueError( "When using list-based iteration, the first yielded value must be a list of column name strings" ) - column_names = cast(list[str], list(first_record)) + column_names = cast(List[str], list(first_record)) all_columns = column_names num_columns = len(column_names) # Get the actual first data record @@ -4386,7 +4434,7 @@ class Table(Queryable): except StopIteration: return self # Only headers, no data if not isinstance(first_record, (list, tuple)): - raise ValueError( # noqa: TRY004 + raise ValueError( "After column names list, all subsequent records must also be lists" ) else: @@ -4396,11 +4444,13 @@ class Table(Queryable): first_record = next(records_iter) except StopIteration: return self - first_record = cast(dict[str, Any], first_record) + first_record = cast(Dict[str, Any], first_record) num_columns = len(first_record.keys()) if num_columns > SQLITE_MAX_VARS: - raise ValueError(f"Rows can have a maximum of {SQLITE_MAX_VARS} columns") + raise ValueError( + "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + ) batch_size = ( 1 if num_columns == 0 @@ -4410,7 +4460,7 @@ class Table(Queryable): self.last_pk = None if truncate and self.exists(): with self.db.atomic(): - self.db.execute(f"DELETE FROM {quote_identifier(self.name)};") + self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) result = None for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) @@ -4423,7 +4473,7 @@ class Table(Queryable): chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] column_types = suggest_column_types(chunk_as_dicts) else: - dict_chunk = cast(list[dict[str, Any]], chunk) + dict_chunk = cast(List[Dict[str, Any]], chunk) column_types = suggest_column_types(dict_chunk) if extracts: for col in extracts: @@ -4450,10 +4500,10 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) else: - all_columns_set: set[str] = set() - for record in cast(list[dict[str, Any]], chunk): + all_columns_set: Set[str] = set() + for record in cast(List[Dict[str, Any]], chunk): all_columns_set.update(record.keys()) - all_columns = sorted(all_columns_set) + all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) if deferred_invalid_pk_check is not None: @@ -4468,7 +4518,7 @@ class Table(Queryable): raise invalid_pk_error else: if not list_mode: - for record in cast(list[dict[str, Any]], chunk): + for record in cast(List[Dict[str, Any]], chunk): all_columns += [ column for column in record if column not in all_columns ] @@ -4507,7 +4557,7 @@ class Table(Queryable): zip(column_names, cast(Sequence[Any], first_record)) ) else: - first_record_dict = cast(dict[str, Any], first_record) + first_record_dict = cast(Dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) elif isinstance(pk, str): @@ -4523,7 +4573,7 @@ class Table(Queryable): # columns so we can report its rowid (and pk if not already # known). Falls back to leaving them unset if the conflict # cannot be resolved to a pk lookup (e.g. a UNIQUE column). - key_cols: list[str] | None = None + key_cols: Optional[List[str]] = None if isinstance(pk, str): key_cols = [pk] elif pk: @@ -4540,10 +4590,12 @@ class Table(Queryable): key_values = None if key_values is not None: where = " and ".join( - f"{quote_identifier(c)} = ?" for c in key_cols + "{} = ?".format(quote_identifier(c)) for c in key_cols ) existing = self.db.execute( - f"select rowid from {quote_identifier(self.name)} where {where} limit 1", + "select rowid from {} where {} limit 1".format( + quote_identifier(self.name), where + ), key_values, ).fetchone() if existing is not None: @@ -4563,9 +4615,7 @@ class Table(Queryable): rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES if (hash_id or (pk and not rowid_pk)) and self.last_rowid: # Set self.last_pk to the pk(s) for that rowid - row = next( - iter(self.rows_where("rowid = ?", [self.last_rowid])) - ) + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: self.last_pk = row[hash_id] elif isinstance(pk, str): @@ -4595,7 +4645,7 @@ class Table(Queryable): for p in pk ) else: - first_record_dict = cast(dict[str, Any], first_record) + first_record_dict = cast(Dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) else: @@ -4653,7 +4703,10 @@ class Table(Queryable): def upsert_all( self, - records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], + records: Union[ + Iterable[Dict[str, Any]], + Iterable[Sequence[Any]], + ], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4691,7 +4744,7 @@ class Table(Queryable): strict=strict, ) - def add_missing_columns(self, records: Iterable[dict[str, Any]]) -> "Table": + def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -4701,17 +4754,17 @@ class Table(Queryable): def lookup( self, - lookup_values: dict[str, Any], - extra_values: dict[str, Any] | None = None, - pk: str | None = "id", - foreign_keys: ForeignKeysType | None = None, - column_order: list[str] | None = None, - not_null: Iterable[str] | None = None, - defaults: dict[str, Any] | None = None, - extracts: dict[str, str] | list[str] | None = None, - conversions: dict[str, str] | None = None, - columns: dict[str, Any] | None = None, - strict: bool | None = False, + lookup_values: Dict[str, Any], + extra_values: Optional[Dict[str, Any]] = None, + pk: Optional[str] = "id", + foreign_keys: Optional[ForeignKeysType] = None, + column_order: Optional[List[str]] = None, + not_null: Optional[Iterable[str]] = None, + defaults: Optional[Dict[str, Any]] = None, + extracts: Optional[Union[Dict[str, str], List[str]]] = None, + conversions: Optional[Dict[str, str]] = None, + columns: Optional[Dict[str, Any]] = None, + strict: Optional[bool] = False, ): """ Create or populate a lookup table with the specified values. @@ -4737,7 +4790,7 @@ class Table(Queryable): :param strict: Boolean, apply STRICT mode if creating the table. """ if not isinstance(lookup_values, dict): - raise ValueError("lookup_values must be a dictionary") # noqa: TRY004 + raise ValueError("lookup_values must be a dictionary") if pk is None: raise ValueError("pk cannot be None") if extra_values is not None and not isinstance(extra_values, dict): @@ -4755,7 +4808,9 @@ class Table(Queryable): } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) # IS rather than = so that null values are matched correctly - wheres = [f"{quote_identifier(column)} IS ?" for column in lookup_values] + wheres = [ + "{} IS ?".format(quote_identifier(column)) for column in lookup_values + ] rows = list( self.rows_where( " and ".join(wheres), [value for _, value in lookup_values.items()] @@ -4795,10 +4850,12 @@ class Table(Queryable): def m2m( self, other_table: Union[str, "Table"], - record_or_iterable: Iterable[dict[str, Any]] | dict[str, Any] | None = None, - pk: Any | Default | None = DEFAULT, - lookup: dict[str, Any] | None = None, - m2m_table: str | None = None, + record_or_iterable: Optional[ + Union[Iterable[Dict[str, Any]], Dict[str, Any]] + ] = None, + pk: Optional[Union[Any, Default]] = DEFAULT, + lookup: Optional[Dict[str, Any]] = None, + m2m_table: Optional[str] = None, alter: bool = False, ): """ @@ -4831,8 +4888,8 @@ class Table(Queryable): raise ValueError("Provide lookup= or record, not both") elif record_or_iterable is None: raise ValueError("Provide lookup= or record, not both") - tables = sorted([self.name, other_table.name]) - columns = [f"{t}_id" for t in tables] + tables = list(sorted([self.name, other_table.name])) + columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: m2m_table_name = m2m_table else: @@ -4842,7 +4899,9 @@ class Table(Queryable): m2m_table_name = candidates[0] elif len(candidates) > 1: raise NoObviousTable( - f"No single obvious m2m table for {self.name}, {other_table.name} - use m2m_table= parameter" + "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( + self.name, other_table.name + ) ) else: # If not, create a new table @@ -4853,7 +4912,7 @@ class Table(Queryable): if isinstance(record_or_iterable, Mapping): records = [record_or_iterable] else: - records = cast(list, record_or_iterable) + records = cast(List, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( @@ -4861,8 +4920,8 @@ class Table(Queryable): ).last_pk m2m_table_obj.insert( { - f"{other_table.name}_id": id, - f"{self.name}_id": our_id, + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, }, replace=True, ) @@ -4870,8 +4929,8 @@ class Table(Queryable): id = other_table.lookup(lookup) m2m_table_obj.insert( { - f"{other_table.name}_id": id, - f"{self.name}_id": our_id, + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, }, replace=True, ) @@ -4918,19 +4977,21 @@ class Table(Queryable): table_quoted = quote_identifier(table) column_quoted = quote_identifier(column) num_null = db.execute( - f"select count(*) from {table_quoted} where {column_quoted} is null" + "select count(*) from {} where {} is null".format( + table_quoted, column_quoted + ) ).fetchone()[0] num_blank = db.execute( - f"select count(*) from {table_quoted} where {column_quoted} = ''" + "select count(*) from {} where {} = ''".format(table_quoted, column_quoted) ).fetchone()[0] num_distinct = db.execute( - f"select count(distinct {column_quoted}) from {table_quoted}" + "select count(distinct {}) from {}".format(column_quoted, table_quoted) ).fetchone()[0] most_common_results = None least_common_results = None if num_distinct == 1: value = db.execute( - f"select {column_quoted} from {table_quoted} limit 1" + "select {} from {} limit 1".format(column_quoted, table_quoted) ).fetchone()[0] most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: @@ -4942,10 +5003,13 @@ class Table(Queryable): most_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - f"select {column_quoted}, count(*) " - f"from {table_quoted} group by {column_quoted} " - f"order by count(*) desc, {column_quoted} " - f"limit {common_limit}" + "select {}, count(*) from {} group by {} order by count(*) desc, {} limit {}".format( + column_quoted, + table_quoted, + column_quoted, + column_quoted, + common_limit, + ) ).fetchall() ] most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) @@ -4957,10 +5021,13 @@ class Table(Queryable): least_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - f"select {column_quoted}, count(*) " - f"from {table_quoted} group by {column_quoted} " - f"order by count(*), {column_quoted} desc " - f"limit {common_limit}" + "select {}, count(*) from {} group by {} order by count(*), {} desc limit {}".format( + column_quoted, + table_quoted, + column_quoted, + column_quoted, + common_limit, + ) ).fetchall() ] least_common_results.sort(key=lambda p: (p[1], p[0])) @@ -5077,7 +5144,7 @@ class View(Queryable): """ try: - self.db.execute(f"DROP VIEW {quote_identifier(self.name)}") + self.db.execute("DROP VIEW {}".format(quote_identifier(self.name))) except sqlite3.OperationalError: if not ignore: raise @@ -5090,14 +5157,16 @@ def jsonify_if_needed(value: object) -> object: return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() - elif isinstance(value, (datetime.timedelta, uuid.UUID)): + elif isinstance(value, datetime.timedelta): + return str(value) + elif isinstance(value, uuid.UUID): return str(value) else: return value def resolve_extracts( - extracts: dict[str, str] | list[str] | tuple[str] | None, + extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]], ) -> dict: if extracts is None: extracts = {} @@ -5108,8 +5177,8 @@ def resolve_extracts( def _decode_default_value(value: str) -> object: if value.startswith("'") and value.endswith("'"): - # It's a string; unescape doubled single quotes - return value[1:-1].replace("''", "'") + # It's a string + return value[1:-1] if value.isdigit(): # It's an integer return int(value) diff --git a/sqlite_utils/hookspecs.py b/sqlite_utils/hookspecs.py index 73d1acc..a746619 100644 --- a/sqlite_utils/hookspecs.py +++ b/sqlite_utils/hookspecs.py @@ -1,7 +1,8 @@ import sqlite3 import click -from pluggy import HookimplMarker, HookspecMarker +from pluggy import HookimplMarker +from pluggy import HookspecMarker hookspec = HookspecMarker("sqlite_utils") hookimpl = HookimplMarker("sqlite_utils") diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 69397ba..00d0fa5 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -1,28 +1,19 @@ -import datetime -from collections.abc import Callable, Iterable +from collections.abc import Iterable from dataclasses import dataclass -from typing import TYPE_CHECKING, Protocol, TypeVar, cast +import datetime +from typing import Callable, cast, TYPE_CHECKING if TYPE_CHECKING: from sqlite_utils.db import Database, Table -class _MigrationFunction(Protocol): - __name__: str - - def __call__(self, db: "Database", /) -> None: ... - - -_MigrationFunctionT = TypeVar("_MigrationFunctionT", bound=_MigrationFunction) - - class Migrations: migrations_table = "_sqlite_migrations" @dataclass class _Migration: name: str - fn: _MigrationFunction + fn: Callable transactional: bool = True @dataclass @@ -41,7 +32,7 @@ class Migrations: def __call__( self, *, name: str | None = None, transactional: bool = True - ) -> Callable[[_MigrationFunctionT], _MigrationFunctionT]: + ) -> Callable: """ :param name: The name to use for this migration - if not provided, the name of the function will be used. @@ -52,11 +43,13 @@ class Migrations: example those that execute ``VACUUM``. """ - def inner(func: _MigrationFunctionT) -> _MigrationFunctionT: - migration_name = name or func.__name__ + def inner(func: Callable) -> Callable: + migration_name = name or getattr(func, "__name__") if any(m.name == migration_name for m in self._migrations): raise ValueError( - f"Migration '{migration_name}' is already registered in set '{self.name}'" + "Migration '{}' is already registered in set '{}'".format( + migration_name, self.name + ) ) self._migrations.append( self._Migration(migration_name, func, transactional) diff --git a/sqlite_utils/plugins.py b/sqlite_utils/plugins.py index 10815b4..0aff7ff 100644 --- a/sqlite_utils/plugins.py +++ b/sqlite_utils/plugins.py @@ -1,7 +1,7 @@ -import sys +from typing import Dict, List, Union import pluggy - +import sys from . import hookspecs pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils") @@ -17,13 +17,13 @@ def ensure_plugins_loaded() -> None: _plugins_loaded = True -def get_plugins() -> list[dict[str, str | list[str]]]: +def get_plugins() -> List[Dict[str, Union[str, List[str]]]]: ensure_plugins_loaded() - plugins: list[dict[str, str | list[str]]] = [] + plugins: List[Dict[str, Union[str, List[str]]]] = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): hookcallers = pm.get_hookcallers(plugin) or [] - plugin_info: dict[str, str | list[str]] = { + plugin_info: Dict[str, Union[str, List[str]]] = { "name": plugin.__name__, "hooks": [h.name for h in hookcallers], } diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index d28a099..55b55a4 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,9 +1,9 @@ from __future__ import annotations -import json -from collections.abc import Callable +from typing import Callable, Optional from dateutil import parser +import json IGNORE: object = object() SET_NULL: object = object() @@ -13,8 +13,8 @@ def parsedate( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: object | None = None, -) -> str | None: + errors: Optional[object] = None, +) -> Optional[str]: """ Parse a date and convert it to ISO date format: yyyy-mm-dd \b @@ -44,8 +44,8 @@ def parsedatetime( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: object | None = None, -) -> str | None: + errors: Optional[object] = None, +) -> Optional[str]: """ Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS \b diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ed5a558..b39b117 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -9,11 +9,20 @@ import itertools import json import os import sys -from collections.abc import Callable, Generator, Iterable, Iterator from typing import ( - TYPE_CHECKING, Any, BinaryIO, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TYPE_CHECKING, TypeVar, Union, cast, @@ -24,8 +33,8 @@ import click from . import recipes if TYPE_CHECKING: - import sqlite3 - from sqlite3 import dbapi2 + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 OperationalError = dbapi2.OperationalError else: @@ -35,7 +44,7 @@ else: OperationalError = dbapi2.OperationalError except ImportError: import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 + from sqlite3 import dbapi2 # noqa: F401 OperationalError = dbapi2.OperationalError @@ -52,8 +61,8 @@ SPATIALITE_PATHS = ( ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() # Type alias for row dictionaries - values can be various SQLite-compatible types -RowValue = None | int | float | str | bytes | bool | list[str] -Row = dict[str, RowValue] +RowValue = Union[None, int, float, str, bytes, bool, List[str]] +Row = Dict[str, RowValue] T = TypeVar("T") @@ -94,7 +103,7 @@ def maximize_csv_field_size_limit() -> None: field_size_limit = int(field_size_limit / 10) -def find_spatialite() -> str | None: +def find_spatialite() -> Optional[str]: """ The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. @@ -123,9 +132,9 @@ def find_spatialite() -> str | None: def suggest_column_types( - records: Iterable[dict[str, Any]], -) -> dict[str, type]: - all_column_types: dict[str, set[type]] = {} + records: Iterable[Dict[str, Any]], +) -> Dict[str, type]: + all_column_types: Dict[str, Set[type]] = {} for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) @@ -133,9 +142,9 @@ def suggest_column_types( def types_for_column_types( - all_column_types: dict[str, set[type]], -) -> dict[str, type]: - column_types: dict[str, type] = {} + all_column_types: Dict[str, Set[type]], +) -> Dict[str, type]: + column_types: Dict[str, type] = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -144,7 +153,7 @@ def types_for_column_types( if {None.__class__} == types: t = str elif len(types) == 1: - t = next(iter(types)) + t = list(types)[0] # But if it's a subclass of list / tuple / dict, use str # instead as we will be storing it as JSON in the table for superclass in (list, tuple, dict): @@ -181,7 +190,7 @@ def column_affinity(column_type: str) -> type: return float -def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]: +def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]: # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them to_fix = [ k @@ -254,9 +263,9 @@ class RowError(Exception): def _extra_key_strategy( - reader: Iterable[dict[str | None, object]], - ignore_extras: bool | None = False, - extras_key: str | None = None, + reader: Iterable[Dict[Optional[str], object]], + ignore_extras: Optional[bool] = False, + extras_key: Optional[str] = None, ) -> Iterable[Row]: # Logic for handling CSV rows with more values than there are headings for row in reader: @@ -270,7 +279,9 @@ def _extra_key_strategy( yield cast(Row, row) elif not extras_key: extras = row.pop(None) - raise RowError(f"Row {row} contained these extra values: {extras}") + raise RowError( + "Row {} contained these extra values: {}".format(row, extras) + ) else: extras_value = row.pop(None) row_out = cast(Row, row) @@ -280,12 +291,12 @@ def _extra_key_strategy( def rows_from_file( fp: BinaryIO, - format: Format | None = None, - dialect: type[csv.Dialect] | None = None, - encoding: str | None = None, - ignore_extras: bool | None = False, - extras_key: str | None = None, -) -> tuple[Iterable[Row], Format]: + format: Optional[Format] = None, + dialect: Optional[Type[csv.Dialect]] = None, + encoding: Optional[str] = None, + ignore_extras: Optional[bool] = False, + extras_key: Optional[str] = None, +) -> Tuple[Iterable[Row], Format]: """ Load a sequence of dictionaries from a file-like object containing one of four different formats. @@ -352,7 +363,7 @@ def rows_from_file( ) return ( _extra_key_strategy( - cast(Iterable[dict[str | None, object]], rows), + cast(Iterable[Dict[Optional[str], object]], rows), ignore_extras, extras_key, ), @@ -368,7 +379,7 @@ def rows_from_file( raise TypeError( "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" ) - if first_bytes.startswith((b"[", b"{")): + if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) else: @@ -382,7 +393,7 @@ def rows_from_file( detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV return ( _extra_key_strategy( - cast(Iterable[dict[str | None, object]], rows), + cast(Iterable[Dict[Optional[str], object]], rows), ignore_extras, extras_key, ), @@ -414,9 +425,9 @@ class TypeTracker: """ def __init__(self) -> None: - self.trackers: dict[str, ValueTracker] = {} + self.trackers: Dict[str, "ValueTracker"] = {} - def wrap(self, iterator: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]: + def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: """ Use this to loop through an existing iterator, tracking the column types as part of the iteration. @@ -430,7 +441,7 @@ class TypeTracker: yield row @property - def types(self) -> dict[str, str]: + def types(self) -> Dict[str, str]: """ A dictionary mapping column names to their detected types. This can be passed to the ``db[table_name].transform(types=tracker.types)`` method. @@ -439,15 +450,17 @@ class TypeTracker: class ValueTracker: - couldbe: dict[str, Callable[[object], bool]] + couldbe: Dict[str, Callable[[object], bool]] def __init__(self) -> None: self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} @classmethod - def get_tests(cls) -> list[str]: + def get_tests(cls) -> List[str]: return [ - key.split("test_")[-1] for key in cls.__dict__ if key.startswith("test_") + key.split("test_")[-1] + for key in cls.__dict__.keys() + if key.startswith("test_") ] def test_integer(self, value: object) -> bool: @@ -479,7 +492,7 @@ class ValueTracker: def evaluate(self, value: object) -> None: if not value or not self.couldbe: return - not_these: list[str] = [] + not_these: List[str] = [] for name, test in self.couldbe.items(): if not test(value): not_these.append(name) @@ -511,14 +524,14 @@ def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None] def _compile_code( code: str, imports: Iterable[str], variable: str = "value" ) -> Callable[..., Any]: - globals_dict: dict[str, Any] = {"r": recipes, "recipes": recipes} + globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes} # Handle imports first so they're available for all approaches for import_ in imports: globals_dict[import_.split(".")[0]] = __import__(import_) # If user defined a convert() function, return that try: - exec(code, globals_dict) # noqa: S102 + exec(code, globals_dict) return cast(Callable[..., object], globals_dict["convert"]) except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass @@ -529,20 +542,20 @@ def _compile_code( fn = eval(code, globals_dict) if callable(fn): return cast(Callable[..., object], fn) - except Exception: # noqa: BLE001, S110 + except Exception: pass # Try compiling their code as a function instead body_variants = [code] # If single line and no 'return', try adding the return if "\n" not in code and not code.strip().startswith("return "): - body_variants.insert(0, f"return {code}") + body_variants.insert(0, "return {}".format(code)) code_o = None for variant in body_variants: - new_code = [f"def fn({variable}):"] + new_code = ["def fn({}):".format(variable)] for line in variant.split("\n"): - new_code.append(f" {line}") + new_code.append(" {}".format(line)) try: code_o = compile("\n".join(new_code), "", "exec") break @@ -553,7 +566,7 @@ def _compile_code( if code_o is None: raise SyntaxError("Could not compile code") - exec(code_o, globals_dict) # noqa: S102 + exec(code_o, globals_dict) return cast(Callable[..., object], globals_dict["fn"]) @@ -569,7 +582,7 @@ def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]: yield itertools.chain([item], itertools.islice(iterator, size - 1)) -def hash_record(record: dict[str, Any], keys: Iterable[str] | None = None) -> str: +def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str: """ ``record`` should be a Python dictionary. Returns a sha1 hash of the keys and values in that record. @@ -590,7 +603,7 @@ def hash_record(record: dict[str, Any], keys: Iterable[str] | None = None) -> st :param record: Record to generate a hash for :param keys: Subset of keys to use for that hash """ - to_hash: dict[str, Any] = record + to_hash: Dict[str, Any] = record if keys is not None: to_hash = {key: record[key] for key in keys} return hashlib.sha1( @@ -600,7 +613,7 @@ def hash_record(record: dict[str, Any], keys: Iterable[str] | None = None) -> st ).hexdigest() -def dedupe_keys(keys: Iterable[str]) -> list[str]: +def dedupe_keys(keys: Iterable[str]) -> List[str]: """ Rename duplicates in a list of column names so every name is unique, by appending ``_2``, ``_3``... to later occurrences - skipping any @@ -623,7 +636,7 @@ def dedupe_keys(keys: Iterable[str]) -> list[str]: new_key = key suffix = 2 while new_key in seen or new_key in taken: - new_key = f"{key}_{suffix}" + new_key = "{}_{}".format(key, suffix) suffix += 1 key = new_key seen.add(key) @@ -631,7 +644,7 @@ def dedupe_keys(keys: Iterable[str]) -> list[str]: return result -def _flatten(d: dict[str, Any]) -> Generator[tuple[str, Any], None, None]: +def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): for key2, value2 in _flatten(value): @@ -640,7 +653,7 @@ def _flatten(d: dict[str, Any]) -> Generator[tuple[str, Any], None, None]: yield key, value -def flatten(row: dict[str, Any]) -> dict[str, Any]: +def flatten(row: Dict[str, Any]) -> Dict[str, Any]: """ Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}`` diff --git a/tests/conftest.py b/tests/conftest.py index a4eb860..728db7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,6 @@ -import pytest - from sqlite_utils import Database from sqlite_utils.utils import sqlite3 +import pytest CREATE_TABLES = """ create table Gosh (c1 text, c2 text, c3 text); @@ -56,7 +55,7 @@ def close_all_databases(): for db in databases: try: db.close() - except sqlite3.Error: + except Exception: pass diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a51bba6..a2ce585 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,10 +1,8 @@ -import sqlite3 - -import pytest -from click.testing import CliRunner - +from sqlite_utils.db import Database, ColumnDetails from sqlite_utils import cli -from sqlite_utils.db import ColumnDetails, Database +from click.testing import CliRunner +import pytest +import sqlite3 @pytest.fixture diff --git a/tests/test_atomic.py b/tests/test_atomic.py index ba16ca5..c3fd02f 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -28,13 +28,11 @@ from sqlite_utils.utils import sqlite3 END; """, [ - ( - "CREATE TRIGGER t_ai AFTER INSERT ON t\n" - " BEGIN\n" - " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" - " INSERT INTO log VALUES ('x;y');\n" - " END;" - ) + "CREATE TRIGGER t_ai AFTER INSERT ON t\n" + " BEGIN\n" + " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" + " INSERT INTO log VALUES ('x;y');\n" + " END;" ], ), ), @@ -51,9 +49,10 @@ def test_atomic_commits(fresh_db): def test_atomic_rolls_back(fresh_db): - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - raise RuntimeError("boom") + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -63,9 +62,10 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): with fresh_db.atomic(): fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) assert list(fresh_db["dogs"].rows) == [ @@ -75,18 +75,20 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): def test_outer_atomic_rolls_back_released_savepoint(fresh_db): - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + with pytest.raises(RuntimeError): with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() def test_executescript_does_not_commit_open_atomic_block(fresh_db): - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db.executescript(""" + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db.executescript(""" CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT); CREATE TRIGGER dogs_ai AFTER INSERT ON dogs BEGIN @@ -95,7 +97,7 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): -- This comment has a semicolon; INSERT INTO dogs VALUES (1, 'Cleo; the first'); """) - raise RuntimeError("boom") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -103,10 +105,11 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): def test_transform_does_not_commit_open_atomic_block(fresh_db): fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) - fresh_db["dogs"].transform(rename={"age": "dog_age"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db["dogs"].transform(rename={"age": "dog_age"}) + raise RuntimeError("boom") assert ( fresh_db["dogs"].schema @@ -146,9 +149,10 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): foreign_keys={"author_id"}, ) - with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + raise RuntimeError("boom") assert ( fresh_db["authors"].schema @@ -350,11 +354,9 @@ def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): # with "cannot rollback - no transaction is active" fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with ( - pytest.raises(sqlite3.IntegrityError, match="trigger says no"), - fresh_db.atomic(), - ): - fresh_db.execute("insert into t (v) values ('bad')") + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction @@ -365,17 +367,16 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( # "no such savepoint" from ROLLBACK TO SAVEPOINT fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with ( - pytest.raises(sqlite3.IntegrityError, match="trigger says no"), - fresh_db.atomic(), - fresh_db.atomic(), - ): - fresh_db.execute("insert into t (v) values ('bad')") + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): fresh_db["t"].insert({"id": 1}, pk="id") - with pytest.raises(sqlite3.IntegrityError), fresh_db.atomic(): - fresh_db.execute("insert or rollback into t (id) values (1)") + with pytest.raises(sqlite3.IntegrityError): + with fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") assert not fresh_db.conn.in_transaction diff --git a/tests/test_cli.py b/tests/test_cli.py index a1e072f..a2135b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,16 +1,14 @@ +from sqlite_utils import cli, Database +from sqlite_utils.db import Index, ForeignKey +from click.testing import CliRunner +from pathlib import Path +import subprocess +import sqlite3 +import sys import json import os -import sqlite3 -import subprocess -import sys -import textwrap -from pathlib import Path - import pytest -from click.testing import CliRunner - -from sqlite_utils import Database, cli -from sqlite_utils.db import ForeignKey, Index +import textwrap def write_json(file_path, data): @@ -23,7 +21,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except sqlite3.DatabaseError: + except Exception: return False finally: db.close() @@ -186,9 +184,9 @@ def test_output_table(db_path, options, expected): db["rows"].insert_all( [ { - "c1": f"verb{i}", - "c2": f"noun{i}", - "c3": f"adjective{i}", + "c1": "verb{}".format(i), + "c2": "noun{}".format(i), + "c3": "adjective{}".format(i), } for i in range(4) ] @@ -680,9 +678,9 @@ def test_optimize(db_path, tables): db[table].insert_all( [ { - "c1": f"verb{i}", - "c2": f"noun{i}", - "c3": f"adjective{i}", + "c1": "verb{}".format(i), + "c2": "noun{}".format(i), + "c3": "adjective{}".format(i), } for i in range(10000) ] @@ -706,9 +704,9 @@ def test_rebuild_fts_fixes_docsize_error(db_path): db = Database(db_path, recursive_triggers=False) records = [ { - "c1": f"verb{i}", - "c2": f"noun{i}", - "c3": f"adjective{i}", + "c1": "verb{}".format(i), + "c2": "noun{}".format(i), + "c3": "adjective{}".format(i), } for i in range(10000) ] @@ -1021,14 +1019,16 @@ def test_query_json_binary(db_path): "data": { "$base64": True, "encoded": ( - "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" - "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" - "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" - "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" - "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" - "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" - "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" - "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" + ( + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" + ) ), }, } @@ -2114,13 +2114,11 @@ _common_other_schema = ( ), ( ["--rename", "name", "name2"], - ( - 'CREATE TABLE "trees" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "address" TEXT,\n' - ' "species_id" INTEGER REFERENCES "species"("id")\n' - ")" - ), + 'CREATE TABLE "trees" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' + ")", 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)', ), ], @@ -2139,9 +2137,9 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): assert result.exit_code == 0 schema = db["trees"].schema assert schema == expected_table_schema - other_schema = next( - t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2") - ).schema + other_schema = [t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2")][ + 0 + ].schema assert other_schema == expected_other_schema @@ -2433,7 +2431,7 @@ def test_long_csv_column_value(tmpdir): with open(csv_path, "w") as csv_file: long_string = "a" * 131073 csv_file.write("id,text\n") - csv_file.write(f"1,{long_string}\n") + csv_file.write("1,{}\n".format(long_string)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "bigtable", csv_path, "--csv"], @@ -2459,8 +2457,8 @@ def test_import_no_headers(tmpdir, args, tsv): csv_path = str(tmpdir / "test.csv") with open(csv_path, "w") as csv_file: sep = "\t" if tsv else "," - csv_file.write(f"Cleo{sep}Dog{sep}5\n") - csv_file.write(f"Tracy{sep}Spider{sep}7\n") + csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep)) + csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep)) result = CliRunner().invoke( cli.cli, ["insert", db_path, "creatures", csv_path] + args + ["--no-detect-types"], @@ -2692,9 +2690,7 @@ def test_integer_overflow_error(tmpdir): def test_python_dash_m(): "Tool can be run using python -m sqlite_utils" result = subprocess.run( - [sys.executable, "-m", "sqlite_utils", "--help"], - stdout=subprocess.PIPE, - check=False, + [sys.executable, "-m", "sqlite_utils", "--help"], stdout=subprocess.PIPE ) assert result.returncode == 0 assert b"Commands for interacting with a SQLite database" in result.stdout @@ -2834,14 +2830,14 @@ def test_load_extension(entrypoint, should_pass, should_fail): for func in should_pass: result = CliRunner().invoke( cli.cli, - ["memory", f"select {func}()", "--load-extension", ext], + ["memory", "select {}()".format(func), "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 0 for func in should_fail: result = CliRunner().invoke( cli.cli, - ["memory", f"select {func}()", "--load-extension", ext], + ["memory", "select {}()".format(func), "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 1 diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 932269b..514f4ac 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -1,13 +1,11 @@ +from click.testing import CliRunner +from sqlite_utils import cli, Database import pathlib +import pytest import subprocess import sys import time -import pytest -from click.testing import CliRunner - -from sqlite_utils import Database, cli - @pytest.fixture def test_db_and_path(tmpdir): diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 65543b1..6c3f5c5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -1,12 +1,10 @@ -import json -import pathlib -import textwrap - -import pytest from click.testing import CliRunner - -import sqlite_utils from sqlite_utils import cli +import sqlite_utils +import json +import textwrap +import pathlib +import pytest @pytest.fixture @@ -52,7 +50,7 @@ def test_convert_code(fresh_db_and_path, code): cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - value = next(iter(db["t"].rows))["text"] + value = list(db["t"].rows)[0]["text"] assert value == "Spooktober" @@ -444,7 +442,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): ) code = "r.jsonsplit(value)" if delimiter: - code = f'recipes.jsonsplit(value, delimiter="{delimiter}")' + code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output @@ -472,7 +470,7 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): ) code = "r.jsonsplit(value)" if type: - code = f"recipes.jsonsplit(value, type={type})" + code = "recipes.jsonsplit(value, type={})".format(type) args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index eefb3fa..df6f80c 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -1,13 +1,11 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner import json +import pytest import subprocess import sys import time -import pytest -from click.testing import CliRunner - -from sqlite_utils import Database, cli - def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") @@ -101,7 +99,7 @@ def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): def test_insert_multiple_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - dogs = [{"id": i, "name": f"Cleo {i}", "age": i + 3} for i in range(1, 21)] + dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) result = CliRunner().invoke( @@ -116,7 +114,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"breed": "mixed", "id": i, "name": f"Cleo {i}", "age": i + 3} + {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21) ] with open(json_path, "w") as fp: @@ -142,7 +140,8 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): def test_insert_not_null_default(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"id": i, "name": f"Cleo {i}", "age": i + 3, "score": 10} for i in range(1, 21) + {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} + for i in range(1, 21) ] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) @@ -588,7 +587,7 @@ def test_insert_streaming_batch_size_1(db_path): return tries += 1 if tries > 10: - assert False, f"Expected {expected}, got {rows}" + assert False, "Expected {}, got {}".format(expected, rows) time.sleep(tries * 0.1) try_until([{"name": "Azi"}]) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 4fb4fb3..2ed4aaa 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,6 +1,5 @@ -import json - import click +import json import pytest from click.testing import CliRunner @@ -29,7 +28,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): fp.write(content) result = CliRunner().invoke( cli.cli, - ["memory", csv_path, f"select * from {sql_from}", "--nl"], + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], input=input, ) assert result.exit_code == 0 @@ -54,7 +53,7 @@ def test_memory_tsv(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, f"select * from {sql_from}"], + ["memory", path, "select * from {}".format(sql_from)], input=input, ) assert result.exit_code == 0, result.output @@ -80,7 +79,7 @@ def test_memory_json(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, f"select * from {sql_from}"], + ["memory", path, "select * from {}".format(sql_from)], input=input, ) assert result.exit_code == 0, result.output @@ -106,7 +105,7 @@ def test_memory_json_nl(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, f"select * from {sql_from}"], + ["memory", path, "select * from {}".format(sql_from)], input=input, ) assert result.exit_code == 0, result.output @@ -136,7 +135,7 @@ def test_memory_csv_encoding(tmpdir, use_stdin): CliRunner() .invoke( cli.cli, - ["memory", csv_path, f"select * from {sql_from}", "--nl"], + ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], input=input, ) .exit_code diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index f49ef10..0f29e36 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -1,8 +1,7 @@ import pathlib -import pytest from click.testing import CliRunner - +import pytest import sqlite_utils import sqlite_utils.cli diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fa23345..fb8f340 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,5 +1,4 @@ import pytest - from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -42,5 +41,5 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): - fresh_db.execute(f"create table foo (col {column_def})") + fresh_db.execute("create table foo (col {})".format(column_def)) assert {"col": expected_type} == fresh_db["foo"].columns_dict diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 4282969..a619fba 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,10 +1,8 @@ -import sys - -import pytest - from sqlite_utils import Database from sqlite_utils.db import TransactionError from sqlite_utils.utils import sqlite3 +import pytest +import sys def test_recursive_triggers(): diff --git a/tests/test_convert.py b/tests/test_convert.py index 879267a..ea3fd96 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,6 +1,5 @@ -import pytest - from sqlite_utils.db import BadMultiValues +import pytest @pytest.mark.parametrize( diff --git a/tests/test_create.py b/tests/test_create.py index 40746bf..d281eb4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,27 +1,25 @@ +from sqlite_utils.db import ( + Index, + Database, + DescIndex, + AlterError, + InvalidColumns, + NoObviousTable, + OperationalError, + ForeignKey, + Table, + View, + NoTable, + NoView, +) +from sqlite_utils.utils import hash_record, sqlite3 import collections import datetime import decimal import json import pathlib -import uuid - import pytest - -from sqlite_utils.db import ( - AlterError, - Database, - DescIndex, - ForeignKey, - Index, - InvalidColumns, - NoObviousTable, - NoTable, - NoView, - OperationalError, - Table, - View, -) -from sqlite_utils.utils import hash_record, sqlite3 +import uuid try: import pandas as pd # type: ignore @@ -701,7 +699,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): "num_columns,should_error", ((900, False), (999, False), (1000, True)) ) def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): - record = {f"c{i}": i for i in range(num_columns)} + record = dict([("c{}".format(i), i) for i in range(num_columns)]) if should_error: with pytest.raises(ValueError): fresh_db["big"].insert(record) @@ -720,9 +718,17 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres records = [ {"c0": "first record"}, # one column in first record -> batch size = 999 # fill out the batch with 99 records with enough columns to exceed THRESHOLD - *[{f"c{i}": j for i in range(extra_columns)} for j in range(batch_size - 1)], + *[ + dict([("c{}".format(i), j) for i in range(extra_columns)]) + for j in range(batch_size - 1) + ], ] - fresh_db["too_many_columns"].insert_all(records, alter=True, batch_size=batch_size) + try: + fresh_db["too_many_columns"].insert_all( + records, alter=True, batch_size=batch_size + ) + except sqlite3.OperationalError: + raise @pytest.mark.parametrize( @@ -904,7 +910,7 @@ def test_insert_list_nested_unicode(fresh_db): def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) - row = next(iter(fresh_db["test"].rows)) + row = list(fresh_db["test"].rows)[0] assert {"uuid"} == row.keys() assert isinstance(row["uuid"], str) assert row["uuid"] == str(uuid4) @@ -912,14 +918,16 @@ def test_insert_uuid(fresh_db): def test_insert_memoryview(fresh_db): fresh_db["test"].insert({"data": memoryview(b"hello")}) - row = next(iter(fresh_db["test"].rows)) + row = list(fresh_db["test"].rows)[0] assert {"data"} == row.keys() assert isinstance(row["data"], bytes) assert row["data"] == b"hello" def test_insert_thousands_using_generator(fresh_db): - fresh_db["test"].insert_all({"i": i, "word": f"word_{i}"} for i in range(10000)) + fresh_db["test"].insert_all( + {"i": i, "word": "word_{}".format(i)} for i in range(10000) + ) assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in fresh_db["test"].columns ] @@ -930,7 +938,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): fresh_db["test"].insert_all( - [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": 101, "extra": "This extra column should cause an exception"}], ) @@ -938,7 +946,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( - [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) @@ -950,7 +958,7 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): # https://github.com/simonw/sqlite-utils/issues/732 fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column") @@ -967,7 +975,7 @@ def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): # known - a pk column that is in neither the table nor the records # still raises fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) @@ -1138,7 +1146,7 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): insert_kwargs = {} else: dogs = fresh_db["dogs"] - insert_kwargs = {"hash_id_columns": ("name", "twitter")} + insert_kwargs = dict(hash_id_columns=("name", "twitter")) id = dogs.insert( {"name": "Cleo", "twitter": "cleopaws", "age": 5}, @@ -1646,7 +1654,7 @@ def test_upsert_uses_pk_from_prior_insert_655(fresh_db): # Upsert should work without specifying pk again table.upsert({"id": 1, "name": "Alice Updated"}) assert table.count == 1 - assert next(iter(table.rows))["name"] == "Alice Updated" + assert list(table.rows)[0]["name"] == "Alice Updated" def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): diff --git a/tests/test_create_view.py b/tests/test_create_view.py index 2b70099..056e246 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -1,5 +1,4 @@ import pytest - from sqlite_utils.utils import OperationalError diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 2815180..3724d99 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -31,7 +31,7 @@ EXAMPLES = [ @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): - fresh_db.execute(f"create table foo (col {column_def})") + fresh_db.execute("create table foo (col {})".format(column_def)) assert initial_value == fresh_db["foo"].columns[0].default_value assert expected_value == fresh_db.quote_default_value( fresh_db["foo"].columns[0].default_value diff --git a/tests/test_delete.py b/tests/test_delete.py index dffb6bb..a2d93aa 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -3,7 +3,7 @@ import sqlite_utils def test_delete_rowid_table(fresh_db): table = fresh_db["table"] - table.insert({"foo": 1}) + table.insert({"foo": 1}).last_pk rowid = table.insert({"foo": 2}).last_pk table.delete(rowid) assert [{"foo": 1}] == list(table.rows) diff --git a/tests/test_docs.py b/tests/test_docs.py index 6bc06c8..f657416 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,10 +1,8 @@ -import re -from pathlib import Path - -import pytest from click.testing import CliRunner - from sqlite_utils import cli, recipes +from pathlib import Path +import pytest +import re docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") @@ -36,7 +34,7 @@ def test_commands_are_documented(documented_commands, command): @pytest.mark.parametrize("command", cli.cli.commands.values()) def test_commands_have_help(command): - assert command.help, f"{command} is missing its help" + assert command.help, "{} is missing its help".format(command) def test_convert_help(): diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index ad853a5..28961d2 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,8 +1,6 @@ -import datetime - -import pytest - from sqlite_utils.db import NoTable +import datetime +import pytest def test_duplicate(fresh_db): @@ -14,7 +12,7 @@ def test_duplicate(fresh_db): "bool_col" INTEGER, "datetime_col" TEXT)""") # Insert one row of mock data: - dt = datetime.datetime.now(datetime.timezone.utc) + dt = datetime.datetime.now() data = { "text_col": "Cleo", "real_col": 3.14, diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 71a8936..2f6b0db 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -1,14 +1,14 @@ -import pytest +from sqlite_utils import Database +from sqlite_utils import cli from click.testing import CliRunner - -from sqlite_utils import Database, cli +import pytest def test_enable_counts_specific_table(fresh_db): foo = fresh_db["foo"] assert fresh_db.table_names() == [] for i in range(10): - foo.insert({"name": f"item {i}"}) + foo.insert({"name": "item {}".format(i)}) assert fresh_db.table_names() == ["foo"] assert foo.count == 10 # Now enable counts @@ -44,7 +44,7 @@ def test_enable_counts_specific_table(fresh_db): assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] # Add some items to test the triggers for i in range(5): - foo.insert({"name": f"item {10 + i}"}) + foo.insert({"name": "item {}".format(10 + i)}) assert foo.count == 15 assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] # Delete some items diff --git a/tests/test_extract.py b/tests/test_extract.py index 915e6e1..c73ee7a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,21 +1,19 @@ -import itertools - -import pytest - from sqlite_utils.db import InvalidColumns +import itertools +import pytest @pytest.mark.parametrize("table", [None, "Species"]) @pytest.mark.parametrize("fk_column", [None, "species"]) def test_extract_single_column(fresh_db, table, fk_column): expected_table = table or "species" - expected_fk = fk_column or f"{expected_table}_id" + expected_fk = fk_column or "{}_id".format(expected_table) iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) fresh_db["tree"].insert_all( ( { "id": i, - "name": f"Tree {i}", + "name": "Tree {}".format(i), "species": next(iter_species), "end": 1, } @@ -28,12 +26,13 @@ def test_extract_single_column(fresh_db, table, fk_column): 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' - f' "{expected_fk}" INTEGER REFERENCES "{expected_table}"("id"),\n' + ' "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table) + ' "end" INTEGER\n' + ")" ) assert fresh_db[expected_table].schema == ( - f'CREATE TABLE "{expected_table}" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + 'CREATE TABLE "{}" (\n'.format(expected_table) + + ' "id" INTEGER PRIMARY KEY,\n' ' "species" TEXT\n' ")" ) @@ -58,7 +57,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): ( { "id": i, - "name": f"Tree {i}", + "name": "Tree {}".format(i), "common_name": next(iter_common), "latin_name": next(iter_latin), } diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 9519b91..7add79a 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,14 +1,13 @@ -import pytest - from sqlite_utils.db import Index +import pytest @pytest.mark.parametrize( "kwargs,expected_table", [ - ({"extracts": {"species_id": "Species"}}, "Species"), - ({"extracts": ["species_id"]}, "species_id"), - ({"extracts": ("species_id",)}, "species_id"), + (dict(extracts={"species_id": "Species"}), "Species"), + (dict(extracts=["species_id"]), "species_id"), + (dict(extracts=("species_id",)), "species_id"), ], ) @pytest.mark.parametrize("use_table_factory", [True, False]) @@ -31,11 +30,15 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): # Should now have two tables: Trees and Species assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( - f'CREATE TABLE "{expected_table}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)' + 'CREATE TABLE "{}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)'.format( + expected_table + ) == fresh_db[expected_table].schema ) assert ( - f'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{expected_table}"("id")\n)' + 'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{}"("id")\n)'.format( + expected_table + ) == fresh_db["Trees"].schema ) # Should have a foreign key reference @@ -48,7 +51,7 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): assert [ Index( seq=0, - name=f"idx_{expected_table}_value", + name="idx_{}_value".format(expected_table), unique=1, origin="c", partial=0, diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 45f4f35..b37d374 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,7 +1,6 @@ """Tests for compound (multi-column) foreign keys - issue #594.""" import pytest - from sqlite_utils import Database from sqlite_utils.db import AlterError, ForeignKey from sqlite_utils.utils import sqlite3 @@ -65,7 +64,7 @@ def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): fresh_db["books"].add_foreign_key("author_id", "authors", "id") fk = fresh_db["books"].foreign_keys[0] with pytest.raises(TypeError): - _table, _column, _other_table, _other_column = fk + table, column, other_table, other_column = fk with pytest.raises(TypeError): fk[0] diff --git a/tests/test_fts.py b/tests/test_fts.py index 50c1770..64ec645 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,9 +1,7 @@ -from unittest.mock import ANY - import pytest - from sqlite_utils import Database from sqlite_utils.utils import sqlite3 +from unittest.mock import ANY search_records = [ { @@ -105,10 +103,9 @@ def test_search_limit_offset(fresh_db): table.enable_fts(["text", "country"], fts_version="FTS4") assert len(list(table.search("are"))) == 2 assert len(list(table.search("are", limit=1))) == 1 - assert next(iter(table.search("are", limit=1, order_by="rowid")))["rowid"] == 1 + assert list(table.search("are", limit=1, order_by="rowid"))[0]["rowid"] == 1 assert ( - next(iter(table.search("are", limit=1, offset=1, order_by="rowid")))["rowid"] - == 2 + list(table.search("are", limit=1, offset=1, order_by="rowid"))[0]["rowid"] == 2 ) @@ -226,20 +223,20 @@ def test_populate_fts_escape_table_names(fresh_db): @pytest.mark.parametrize("fts_version", ("4", "5")) def test_fts_tokenize(fresh_db, fts_version): - table_name = f"searchable_{fts_version}" + table_name = "searchable_{}".format(fts_version) table = fresh_db[table_name] table.insert_all(search_records) # Test without porter stemming table.enable_fts( ["text", "country"], - fts_version=f"FTS{fts_version}", + fts_version="FTS{}".format(fts_version), ) assert [] == list(table.search("bite")) # Test WITH stemming table.disable_fts() table.enable_fts( ["text", "country"], - fts_version=f"FTS{fts_version}", + fts_version="FTS{}".format(fts_version), tokenize="porter", ) rows = list(table.search("bite", order_by="rowid")) @@ -254,10 +251,10 @@ def test_fts_tokenize(fresh_db, fts_version): def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): - table_name = f"searchable_{fts_version}" + table_name = "searchable_{}".format(fts_version) table = fresh_db[table_name] table.insert_all(search_records) - table.enable_fts(["text", "country"], fts_version=f"FTS{fts_version}") + table.enable_fts(["text", "country"], fts_version="FTS{}".format(fts_version)) # You can call optimize successfully against the tables OR their _fts equivalents: for table_name in ( "searchable_4", @@ -313,12 +310,12 @@ def test_disable_fts(fresh_db, create_triggers): expected_triggers = {"searchable_ai", "searchable_ad", "searchable_au"} else: expected_triggers = set() - assert expected_triggers == { + assert expected_triggers == set( r[0] for r in fresh_db.execute( "select name from sqlite_master where type = 'trigger'" ).fetchall() - } + ) # Now run .disable_fts() and confirm it worked table.disable_fts() assert ( @@ -427,7 +424,7 @@ def test_enable_fts_replace(kwargs): db["books"].enable_fts(**kwargs, replace=True) # Check that the new configuration is correct if should_have_changed_columns: - assert db["books_fts"].columns_dict.keys() == {"title"} + assert db["books_fts"].columns_dict.keys() == set(["title"]) if "create_triggers" in kwargs: assert db["books"].triggers if "fts_version" in kwargs: @@ -744,7 +741,6 @@ def test_enable_fts_cli_on_view_errors(tmpdir): db.create_view("v", "select * from t") db.close() from click.testing import CliRunner - from sqlite_utils import cli as cli_module result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"]) diff --git a/tests/test_get.py b/tests/test_get.py index 3cdaed8..63c4a2e 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -1,5 +1,4 @@ import pytest - from sqlite_utils.db import NotFoundError diff --git a/tests/test_gis.py b/tests/test_gis.py index 8b41d22..f39554e 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -1,8 +1,7 @@ import json - import pytest -from click.testing import CliRunner +from click.testing import CliRunner from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 @@ -105,7 +104,7 @@ def test_query_load_extension(use_spatialite_shortcut): [ ":memory:", "select spatialite_version()", - f"--load-extension={load_extension}", + "--load-extension={}".format(load_extension), ], ) assert result.exit_code == 0, result.stdout diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index ab652c7..f12f865 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -1,6 +1,5 @@ -import hypothesis.strategies as st from hypothesis import given - +import hypothesis.strategies as st import sqlite_utils diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1724d2d..88e49a8 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -1,11 +1,9 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner import os import pathlib -import sys - import pytest -from click.testing import CliRunner - -from sqlite_utils import Database, cli +import sys @pytest.mark.parametrize("silent", (False, True)) @@ -46,7 +44,7 @@ def test_insert_files(silent, pk_args, expected_pks): ) cols = [] for coltype in coltypes: - cols += ["-c", f"{coltype}:{coltype}"] + cols += ["-c", "{}:{}".format(coltype, coltype)] result = runner.invoke( cli.cli, ["insert-files", db_path, "files", str(tmpdir)] @@ -144,7 +142,7 @@ def test_insert_files_stdin(use_text, encoding, input, expected): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - row = next(iter(db["files"].rows)) + row = list(db["files"].rows)[0] key = "content" if use_text: key = "content_text" @@ -169,5 +167,5 @@ def test_insert_files_bad_text_encoding_error(): ) assert result.exit_code == 1, result.output assert result.output.strip().startswith( - f"Error: Could not read file '{latin.resolve()!s}' as text" + "Error: Could not read file '{}' as text".format(str(latin.resolve())) ) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b7e8fc2..8b6765d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,7 +1,6 @@ +from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest -from sqlite_utils.db import Database, Index, View, XIndex, XIndexColumn - def _check_supports_strict(): """Check if SQLite supports strict tables without leaking the database.""" @@ -58,8 +57,8 @@ def test_detect_fts_similar_tables(fresh_db, reverse_order): fresh_db[table2].insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - assert fresh_db[table1].detect_fts() == f"{table1}_fts" - assert fresh_db[table2].detect_fts() == f"{table2}_fts" + assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) + assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) def test_tables(existing_db): @@ -312,7 +311,6 @@ def test_table_strict(fresh_db, create_table, expected_strict): 1, 1.3, "foo", - "O'Brien", True, b"binary", ), @@ -325,16 +323,6 @@ def test_table_default_values(fresh_db, value): assert default_values == {"value": value} -def test_table_default_values_escaped_quotes(fresh_db): - # SQLite stores string defaults with single quotes doubled, so - # introspection needs to unescape them again - fresh_db.execute( - "create table t (id integer primary key, name text default 'O''Brien')" - ) - assert "default 'O''Brien'" in fresh_db["t"].schema - assert fresh_db["t"].default_values == {"name": "O'Brien"} - - def test_pks_use_primary_key_declaration_order(fresh_db): # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - # pks must follow the declaration order, which is what SQLite uses to diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index 646098e..746c9c1 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -3,7 +3,6 @@ Tests for list-based iteration in insert_all and upsert_all """ import pytest - from sqlite_utils import Database diff --git a/tests/test_lookup.py b/tests/test_lookup.py index c93d1ed..da4f18b 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -1,6 +1,5 @@ -import pytest - from sqlite_utils.db import Index +import pytest def test_lookup_new_table(fresh_db): diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 4fca918..d613bb9 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -1,6 +1,5 @@ -import pytest - from sqlite_utils.db import ForeignKey, NoObviousTable +import pytest def test_insert_m2m_single(fresh_db): @@ -66,7 +65,8 @@ def test_insert_m2m_iterable(fresh_db): iterable_records = ({"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}) def iterable(): - yield from iterable_records + for record in iterable_records: + yield record platypuses = fresh_db["platypuses"] platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 3f3dfea..04185fc 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,5 +1,4 @@ import pytest - import sqlite_utils from sqlite_utils import Migrations @@ -155,9 +154,10 @@ def test_non_transactional_migration_allows_vacuum(tmpdir): def test_apply_composes_inside_outer_transaction(migrations): db = sqlite_utils.Database(memory=True) - with pytest.raises(ZeroDivisionError), db.atomic(): - migrations.apply(db) - raise ZeroDivisionError + with pytest.raises(ZeroDivisionError): + with db.atomic(): + migrations.apply(db) + raise ZeroDivisionError # The outer transaction rolled back, taking the migrations with it assert db.table_names() == [] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index ef202be..c793e32 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,12 +1,9 @@ -import importlib -import sqlite3 -import sys - -import click -import pytest from click.testing import CliRunner - -from sqlite_utils import Database, cli, hookimpl, plugins +import click +import importlib +import pytest +import sys +from sqlite_utils import cli, Database, hookimpl, plugins def _supports_pragma_function_list(): @@ -14,7 +11,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except sqlite3.DatabaseError: + except Exception: return False finally: db.close() diff --git a/tests/test_query.py b/tests/test_query.py index 9d79755..06847da 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,6 +1,5 @@ -import types - import pytest +import types from sqlite_utils.utils import sqlite3 diff --git a/tests/test_recipes.py b/tests/test_recipes.py index c6222a3..a7c7ef7 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,9 +1,7 @@ -import json - -import pytest - from sqlite_utils import recipes from sqlite_utils.utils import sqlite3 +import json +import pytest @pytest.fixture diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 09e237e..bce53d5 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -1,9 +1,7 @@ -import pathlib -import sqlite3 - -import pytest - from sqlite_utils import Database +import sqlite3 +import pathlib +import pytest def test_recreate_ignored_for_in_memory(): diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index 8c080d6..a19fed6 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -1,9 +1,7 @@ +from sqlite_utils.utils import rows_from_file, Format, RowError from io import BytesIO, StringIO - import pytest -from sqlite_utils.utils import Format, RowError, rows_from_file - @pytest.mark.parametrize( "input,expected_format", @@ -31,7 +29,7 @@ def test_rows_from_file_detect_format(input, expected_format): ) def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): try: - rows, _format = rows_from_file( + rows, format = rows_from_file( BytesIO(b"id,name\r\n1,Cleo,oops"), format=Format.CSV, ignore_extras=ignore_extras, diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 7149978..4bbdb66 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -1,9 +1,7 @@ -import pathlib - -import pytest +from sqlite_utils import cli, Database from click.testing import CliRunner - -from sqlite_utils import Database, cli +import pathlib +import pytest sniff_dir = pathlib.Path(__file__).parent / "sniff" diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index d4f28d3..e36c58f 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -1,7 +1,5 @@ -from collections import OrderedDict - import pytest - +from collections import OrderedDict from sqlite_utils.utils import suggest_column_types diff --git a/tests/test_tracer.py b/tests/test_tracer.py index ec81f2f..d14697d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -53,18 +53,16 @@ def test_with_tracer(): assert len(collected) == 4 assert collected == [ ( - ( - "SELECT name FROM sqlite_master\n" - " WHERE rootpage = 0\n" - " AND (\n" - " sql LIKE :like\n" - " OR sql LIKE :like2\n" - " OR (\n" - " tbl_name = :table\n" - " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" - " )\n" - " )" - ), + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE :like\n" + " OR sql LIKE :like2\n" + " OR (\n" + " tbl_name = :table\n" + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )", { "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', @@ -74,23 +72,21 @@ def test_with_tracer(): ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - ( - 'with "original" as (\n' - " select\n" - " rowid,\n" - " *\n" - ' from "dogs"\n' - ")\n" - "select\n" - ' "original".*\n' - "from\n" - ' "original"\n' - ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' - "where\n" - ' "dogs_fts" match :query\n' - "order by\n" - ' "dogs_fts".rank' - ), + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "dogs"\n' + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' + "where\n" + ' "dogs_fts" match :query\n' + "order by\n" + ' "dogs_fts".rank', {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index b9ee126..f0f5019 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,9 +1,8 @@ import sqlite3 -import pytest - -from sqlite_utils.db import ForeignKey, TransactionError, TransformError +from sqlite_utils.db import ForeignKey, TransformError from sqlite_utils.utils import OperationalError +import pytest @pytest.mark.parametrize( @@ -114,7 +113,7 @@ def test_transform_sql_table_with_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -187,7 +186,7 @@ def test_transform_sql_table_with_no_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) - sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) + sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -433,163 +432,6 @@ def test_transform_verify_foreign_keys(fresh_db): assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] -@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) -def test_transform_on_delete_cascade_does_not_delete_records( - fresh_db, use_pragma_foreign_keys -): - # Transforming a table drops and recreates it - if another table references - # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must - # not cascade and delete the referencing records - if use_pragma_foreign_keys: - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - # Transform the table on the other end of the cascading foreign key - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ - {"id": 1, "author_name": "Ursula K. Le Guin"} - ] - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - # Transforming the table with the cascading foreign key should not - # delete its records either - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "book_title": "The Dispossessed", "author_id": 1} - ] - if use_pragma_foreign_keys: - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"]) -def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete): - # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a - # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign - # keys inside an open transaction would fire those actions when the old - # table is dropped - transform() should refuse instead - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(f""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE {on_delete} - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - previous_schema = fresh_db["authors"].schema - with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) - message = str(excinfo.value) - assert "books" in message - assert f"ON DELETE {on_delete.upper()}" in message - # Nothing should have changed - assert fresh_db["authors"].schema == previous_schema - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): - # The copied table carries a foreign key referencing the original table - # name, so a self-referential cascade would wipe the copy too - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE categories ( - id INTEGER PRIMARY KEY, - name TEXT, - parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE - ); - """) - fresh_db["categories"].insert_all( - [ - {"id": 1, "name": "Fiction", "parent_id": None}, - {"id": 2, "name": "Science Fiction", "parent_id": 1}, - ] - ) - with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) - assert "categories" in str(excinfo.value) - assert fresh_db["categories"].count == 2 - - -def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): - # An inbound foreign key without a destructive ON DELETE action is safe - # inside a transaction thanks to PRAGMA defer_foreign_keys - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ - {"id": 1, "author_name": "Ursula K. Le Guin"} - ] - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] - - -def test_transform_in_transaction_allowed_for_child_table(fresh_db): - # The table being transformed only has an outbound foreign key - dropping - # it fires no ON DELETE actions, so this is allowed inside a transaction - fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "book_title": "The Dispossessed", "author_id": 1} - ] - - -def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): - # With PRAGMA foreign_keys off (the default) no cascades can fire, so - # transform inside a transaction is safe even with a CASCADE schema - fresh_db.executescript(""" - CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); - CREATE TABLE books ( - id INTEGER PRIMARY KEY, - title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE - ); - """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["books"].rows) == [ - {"id": 1, "title": "The Dispossessed", "author_id": 1} - ] - - def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) @@ -714,15 +556,15 @@ def test_transform_preserves_rowids(fresh_db, table_type): # Now delete and insert a row to mix up the `rowid` sequence fresh_db["places"].delete_where("id = ?", ["2"]) fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) - previous_rows = [ + previous_rows = list( tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ] + ) # Transform it fresh_db["places"].transform(column_order=("country", "name")) # Should be the same - next_rows = [ + next_rows = list( tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ] + ) assert previous_rows == next_rows diff --git a/tests/test_update.py b/tests/test_update.py index e6ae7d8..03bec11 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -43,7 +43,7 @@ def test_update_compound_pk_table(fresh_db): ) def test_update_invalid_pk(fresh_db, pk, update_pk): table = fresh_db["table"] - table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk) + table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk).last_pk with pytest.raises(NotFoundError): table.update(update_pk, {"v": 2}) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 0eaae9b..a782b26 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -1,7 +1,6 @@ -import pytest - -from sqlite_utils import Database from sqlite_utils.db import PrimaryKeyRequired +from sqlite_utils import Database +import pytest @pytest.mark.parametrize("use_old_upsert", (False, True)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 360a443..3de5e94 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,10 +1,8 @@ +from sqlite_utils import utils import csv import io - import pytest -from sqlite_utils import utils - @pytest.mark.parametrize( "input,expected,should_be_is", @@ -59,7 +57,7 @@ def test_maximize_csv_field_size_limit(): # Reset to default in case other tests have changed it csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) long_value = "a" * 131073 - long_csv = f"id,text\n1,{long_value}" + long_csv = "id,text\n1,{}".format(long_value) fp = io.BytesIO(long_csv.encode("utf-8")) # Using rows_from_file should error with pytest.raises(csv.Error): diff --git a/tests/test_wal.py b/tests/test_wal.py index 35318f8..2ddcf54 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,5 +1,4 @@ import pytest - from sqlite_utils import Database from sqlite_utils.db import TransactionError @@ -12,7 +11,7 @@ def db_path_tmpdir(tmpdir): def test_enable_disable_wal(db_path_tmpdir): - db, _path, tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir assert len(tmpdir.listdir()) == 1 assert "delete" == db.journal_mode assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] @@ -26,11 +25,12 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): - db, _path, _tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.enable_wal() + with pytest.raises(TransactionError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.enable_wal() # The atomic() block must have rolled back cleanly and the # journal mode must be unchanged assert db.journal_mode == "delete" @@ -38,18 +38,19 @@ def test_enable_wal_inside_transaction_raises(db_path_tmpdir): def test_disable_wal_inside_transaction_raises(db_path_tmpdir): - db, _path, _tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir db.enable_wal() db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.disable_wal() + with pytest.raises(TransactionError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.disable_wal() assert db.journal_mode == "wal" assert [r["id"] for r in db["test"].rows] == [1] def test_ensure_autocommit_on(db_path_tmpdir): - db, _path, _tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir previous_isolation_level = db.conn.isolation_level assert previous_isolation_level is not None with db.ensure_autocommit_on(): @@ -62,7 +63,7 @@ def test_ensure_autocommit_on(db_path_tmpdir): def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction - db, _path, _tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir db.enable_wal() with db.atomic(): db["test"].insert({"id": 1}, pk="id") @@ -74,12 +75,13 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # Setting isolation_level commits any pending transaction as a side # effect, silently breaking the caller's rollback guarantee - so # entering autocommit mode with a transaction open is an error - db, _path, _tmpdir = db_path_tmpdir + db, path, tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") db.begin() db.execute("insert into test (id) values (2)") - with pytest.raises(TransactionError), db.ensure_autocommit_on(): - pass + with pytest.raises(TransactionError): + with db.ensure_autocommit_on(): + pass # The transaction is still open and can still be rolled back assert db.conn.in_transaction db.rollback()