diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 587380ed..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -89,8 +89,7 @@ def pytest_runtest_protocol(item, nextitem): continue try: ds.close() - except Exception as e: # noqa: BLE001 - # Surfaced as a pytest warning; teardown must not fail the run + except Exception as e: item.warn( pytest.PytestUnraisableExceptionWarning( f"Error closing Datasette instance: {e!r}" diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 7503f1d5..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,9 +1,7 @@ -import time - -from itsdangerous import BadSignature - from datasette import hookimpl +from itsdangerous import BadSignature from datasette.utils import baseconv +import time @hookimpl diff --git a/datasette/app.py b/datasette/app.py index c82ea075..0e31273d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,8 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -13,10 +12,11 @@ import dataclasses import datetime import functools import glob +import httpx import importlib.metadata import inspect +from itsdangerous import BadSignature import json -import logging import os import re import secrets @@ -28,41 +28,90 @@ import urllib.parse from concurrent import futures from pathlib import Path -import httpx -from itsdangerous import BadSignature, URLSafeSerializer +from markupsafe import Markup, escape +from itsdangerous import URLSafeSerializer from jinja2 import ( ChoiceLoader, Environment, FileSystemLoader, - PrefixLoader, pass_context, + PrefixLoader, ) from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from markupsafe import Markup, escape -from . import stored_queries, write_sql -from .column_types import SQLiteType -from .csrf import CrossOriginProtectionMiddleware -from .database import Database, QueryInterrupted from .events import Event -from .plugins import DEFAULT_PLUGINS, get_plugins, pm +from .column_types import SQLiteType +from . import stored_queries, write_sql +from .views import Context +from .views.database import ( + database_download, + DatabaseView, + QueryView, +) +from .views.table_create_alter import ( + DatabaseForeignKeyTargetsView, + TableAlterView, + TableCreateView, + TableForeignKeySuggestionsView, +) +from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView +from .views.stored_queries import ( + QueryCreateAnalyzeView, + QueryDeleteView, + QueryDefinitionView, + QueryEditView, + GlobalQueryListView, + QueryListView, + QueryParametersView, + QueryStoreView, + QueryUpdateView, +) +from .views.index import IndexView +from .views.special import ( + JsonDataView, + PatternPortfolioView, + AutocompleteDebugView, + AuthTokenView, + ApiExplorerView, + CreateTokenView, + LogoutView, + AllowDebugView, + PermissionsDebugView, + MessagesDebugView, + AllowedResourcesView, + PermissionRulesView, + PermissionCheckView, + JumpView, + InstanceSchemaView, + DatabaseSchemaView, + TableSchemaView, +) +from .views.table import ( + TableAutocompleteView, + TableInsertView, + TableUpsertView, + TableSetColumnTypeView, + TableDropView, + TableFragmentView, + table_view, +) +from .views.row import RowView, RowDeleteView, RowUpdateView from .renderer import json_renderer -from .resources import DatabaseResource, TableResource -from .tokens import TokenInvalid -from .tracer import AsgiTracer from .url_builder import Urls +from .database import Database, QueryInterrupted + from .utils import ( - SPATIALITE_FUNCTIONS, PaginatedResources, PrefixedUrlString, + SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, baseconv, call_with_supported_arguments, detect_json1, + add_cors_headers, display_actor, escape_css_string, escape_sqlite, @@ -72,97 +121,47 @@ from .utils import ( move_plugins_and_allow, move_table_config, parse_metadata, - redact_keys, resolve_env_secrets, resolve_routes, - row_sql_params_pks, sha256_file, tilde_decode, tilde_encode, to_css_class, urlsafe_components, + redact_keys, + row_sql_params_pks, ) +from .tokens import TokenInvalid from .utils.asgi import ( AsgiLifespan, - AsgiRunOnFirstRequest, BadRequest, - DatabaseNotFound, Forbidden, NotFound, + DatabaseNotFound, + TableNotFound, + RowNotFound, Request, Response, - RowNotFound, - TableNotFound, + AsgiRunOnFirstRequest, + asgi_static, asgi_send, asgi_send_file, asgi_send_redirect, - asgi_static, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, ) +from .tracer import AsgiTracer +from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -from .views import Context -from .views.database import ( - DatabaseView, - QueryView, - database_download, -) -from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView -from .views.index import IndexView -from .views.row import RowDeleteView, RowUpdateView, RowView -from .views.special import ( - AllowDebugView, - AllowedResourcesView, - ApiExplorerView, - AuthTokenView, - AutocompleteDebugView, - CreateTokenView, - DatabaseSchemaView, - InstanceSchemaView, - JsonDataView, - JumpView, - LogoutView, - MessagesDebugView, - PatternPortfolioView, - PermissionCheckView, - PermissionRulesView, - PermissionsDebugView, - TableSchemaView, -) -from .views.stored_queries import ( - GlobalQueryListView, - QueryCreateAnalyzeView, - QueryDefinitionView, - QueryDeleteView, - QueryEditView, - QueryListView, - QueryParametersView, - QueryStoreView, - QueryUpdateView, -) -from .views.table import ( - TableAutocompleteView, - TableDropView, - TableFragmentView, - TableInsertView, - TableSetColumnTypeView, - TableUpsertView, - table_view, -) -from .views.table_create_alter import ( - DatabaseForeignKeyTargetsView, - TableAlterView, - TableCreateView, - TableForeignKeySuggestionsView, -) + +from .resources import DatabaseResource, TableResource app_root = Path(__file__).parent.parent -logger = logging.getLogger(__name__) - # Context variable to track when code is executing within a datasette.client request _in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) @@ -185,7 +184,7 @@ class PermissionCheck: """Represents a logged permission check for debugging purposes.""" when: str - actor: dict[str, Any] | None + actor: Dict[str, Any] | None action: str parent: str | None child: str | None @@ -435,7 +434,7 @@ class Datasette: if config_dir: db_files = [] for ext in ("db", "sqlite", "sqlite3"): - db_files.extend(config_dir.glob(f"*.{ext}")) + db_files.extend(config_dir.glob("*.{}".format(ext))) self.files += tuple(str(f) for f in db_files) if ( config_dir @@ -676,10 +675,10 @@ class Datasette: def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: - for hook_environment in pm.hook.jinja2_environment_from_request( + for environment in pm.hook.jinja2_environment_from_request( datasette=self, request=request, env=environment ): - environment = hook_environment + pass return environment def get_action(self, name_or_abbr: str): @@ -733,7 +732,7 @@ class Datasette: catalog_database_names.update( row["database_name"] for row in await internal_db.execute( - f"select distinct database_name from {table}" + "select distinct database_name from {}".format(table) ) if row["database_name"] is not None ) @@ -744,7 +743,7 @@ class Datasette: for stale_db_name in stale_databases: for table in catalog_table_names: conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", + "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], ) @@ -793,13 +792,17 @@ class Datasette: action.name in action_names and action != action_names[action.name] ): - raise StartupError(f"Duplicate action name: {action.name}") + raise StartupError( + "Duplicate action name: {}".format(action.name) + ) if ( action.abbr and action.abbr in action_abbrs and action != action_abbrs[action.abbr] ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") + raise StartupError( + "Duplicate action abbr: {}".format(action.abbr) + ) action_names[action.name] = action if action.abbr: action_abbrs[action.abbr] = action @@ -858,7 +861,7 @@ class Datasette: actor_id: str, *, expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + restrictions: "TokenRestrictions | None" = None, handler: str | None = None, ) -> str: """ @@ -915,7 +918,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - name = next(iter(self.databases.keys())) + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -928,7 +931,7 @@ class Datasette: suggestion = name i = 2 while name in self.databases: - name = f"{suggestion}_{i}" + name = "{}_{}".format(suggestion, i) i += 1 db.name = name db.route = route or name @@ -963,14 +966,13 @@ class Datasette: for db in dbs: try: db.close() - except Exception as e: # noqa: BLE001 - # Collect the first failure and re-raise after every close() has run + except Exception as e: if first_exception is None: first_exception = e if self.executor is not None: try: self.executor.shutdown(wait=True, cancel_futures=True) - except Exception as e: # noqa: BLE001 + except Exception as e: if first_exception is None: first_exception = e if first_exception is not None: @@ -1319,15 +1321,24 @@ class Datasette: actual = ( actual_sqlite_type.value if actual_sqlite_type is not None - else f"unrecognized {column_detail.type!r}" + else "unrecognized {!r}".format(column_detail.type) ) raise ValueError( - f"Column type {ct_cls.name!r} is only applicable to SQLite types {allowed} but {database}.{resource}.{column} " - f"has SQLite type {actual}" + "Column type {!r} is only applicable to SQLite types {} but {}.{}.{} " + "has SQLite type {}".format( + ct_cls.name, + allowed, + database, + resource, + column, + actual, + ) ) async def _apply_column_types_config(self): """Load column_types from datasette.json config into the internal DB.""" + import logging + for db_name, db_conf in (self.config or {}).get("databases", {}).items(): for table_name, table_conf in db_conf.get("tables", {}).items(): for col_name, ct in table_conf.get("column_types", {}).items(): @@ -1337,7 +1348,7 @@ class Datasette: col_type = ct["type"] config = ct.get("config") if col_type not in self._column_types: - logger.warning( + logging.warning( "column_types config references unknown type %r " "for %s.%s.%s", col_type, @@ -1350,7 +1361,7 @@ class Datasette: db_name, table_name, col_name, col_type, config ) except ValueError as ex: - logger.warning(str(ex)) + logging.warning(str(ex)) async def get_column_type(self, database: str, resource: str, column: str): """ @@ -1403,7 +1414,7 @@ class Datasette: resource: str, column: str, column_type: str, - config: dict | None = None, + config: dict = None, ) -> None: """Assign a column type. Overwrites any existing assignment.""" ct_cls = self._column_types.get(column_type) @@ -1487,7 +1498,9 @@ class Datasette: possible_names = {plugin["name"], plugin["name"].replace("-", "_")} if plugin_name in possible_names: return _resolve_static_asset_path(plugin["static_path"], path) - raise FileNotFoundError(f"No static assets found for plugin {plugin_name}") + raise FileNotFoundError( + "No static assets found for plugin {}".format(plugin_name) + ) def _static_mounted_asset(self, mount_name, path): mount_name = mount_name.strip("/") @@ -1497,7 +1510,7 @@ class Datasette: _resolve_static_asset_path(dirname, path), self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))), ) - raise FileNotFoundError(f"No static mount found for {mount_name}") + raise FileNotFoundError("No static mount found for {}".format(mount_name)) def _static_asset_hash(self, filepath): filepath = Path(filepath) @@ -1588,17 +1601,18 @@ class Datasette: if await self.allowed(action="view-instance", actor=actor): crumbs.append({"href": self.urls.instance(), "label": "home"}) # Database link - if database and await self.allowed( - action="view-database", - resource=DatabaseResource(database=database), - actor=actor, - ): - crumbs.append( - { - "href": self.urls.database(database), - "label": database, - } - ) + if database: + if await self.allowed( + action="view-database", + resource=DatabaseResource(database=database), + actor=actor, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) # Table link if table: assert database, "table= requires database=" @@ -1617,7 +1631,7 @@ class Datasette: async def actors_from_ids( self, actor_ids: Iterable[str | int] - ) -> dict[int | str, dict]: + ) -> Dict[int | str, Dict]: result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) if result is None: # Do the default thing @@ -1626,9 +1640,9 @@ class Datasette: return result async def track_event(self, event: Event): - assert isinstance( - event, self.event_classes - ), f"Invalid event type: {type(event)}" + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) for hook in pm.hook.track_event(datasette=self, event=event): await await_me_maybe(hook) @@ -1665,7 +1679,7 @@ class Datasette: self, actor: dict, action: str, - resource: Resource | None = None, + resource: "Resource" | None = None, ): """ Check if actor can see a resource and if it's private. @@ -1864,7 +1878,10 @@ class Datasette: if truncated and resources: last_resource = resources[-1] # Use tilde-encoding like table pagination - next_token = f"{tilde_encode(str(last_resource.parent))},{tilde_encode(str(last_resource.child))}" + next_token = "{},{}".format( + tilde_encode(str(last_resource.parent)), + tilde_encode(str(last_resource.child)), + ) return PaginatedResources( resources=resources, @@ -1882,7 +1899,7 @@ class Datasette: self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> bool: """ @@ -1913,7 +1930,7 @@ class Datasette: self, *, actions: Sequence[str], - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ) -> dict[str, bool]: """ @@ -1934,11 +1951,11 @@ class Datasette: ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ + from datasette.utils.actions_sql import check_permissions_for_actions from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, ) - from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None @@ -2027,7 +2044,7 @@ class Datasette: self, *, action: str, - resource: Resource = None, + resource: "Resource" = None, actor: dict | None = None, ): """ @@ -2081,15 +2098,13 @@ class Datasette: db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) # Find the foreign_key for this column - fk = next( - ( + try: + fk = [ foreign_key for foreign_key in foreign_keys if foreign_key["column"] == column - ), - None, - ) - if fk is None: + ][0] + except IndexError: return {} # Ensure user has permission to view the referenced table from datasette.resources import TableResource @@ -2177,17 +2192,16 @@ class Datasette: sqlite_extensions[extension] = result.fetchone()[0] else: sqlite_extensions[extension] = None - except Exception: # noqa: BLE001, S110 - # Probing for optional SQLite extensions - absence is the normal case + except Exception: pass # More details on SpatiaLite if "spatialite" in sqlite_extensions: spatialite_details = {} for fn in SPATIALITE_FUNCTIONS: try: - result = conn.execute(f"select {fn}()") + result = conn.execute("select {}()".format(fn)) spatialite_details[fn] = result.fetchone()[0] - except sqlite3.Error as e: + except Exception as e: spatialite_details[fn] = {"error": str(e)} sqlite_extensions["spatialite"] = spatialite_details @@ -2195,7 +2209,9 @@ class Datasette: fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): try: - conn.execute(f"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)") + conn.execute( + "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) + ) fts_versions.append(fts) except sqlite3.OperationalError: continue @@ -2254,7 +2270,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": sorted(set(p["hooks"])), + "hooks": list(sorted(set(p["hooks"]))), } for p in ps ] @@ -2330,15 +2346,13 @@ class Datasette: async def render_template( self, - templates: list[str] | str | Template, - context: dict[str, Any] | Context | None = None, + templates: List[str] | str | Template, + context: Dict[str, Any] | Context | None = None, request: Request | None = None, view_name: str | None = None, ): if not self._startup_invoked: - raise RuntimeError( - "render_template() called before await ds.invoke_startup()" - ) + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -2384,9 +2398,9 @@ class Datasette: datasette=self, ): extra_vars = await await_me_maybe(extra_vars) - assert isinstance( - extra_vars, dict - ), f"extra_vars is of type {type(extra_vars)}" + assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( + type(extra_vars) + ) extra_template_vars.update(extra_vars) async def menu_links(): @@ -2405,27 +2419,29 @@ class Datasette: # the contract tests fail otherwise template_context = { **context, - "request": request, - "crumb_items": self._crumb_items, - "urls": self.urls, - "actor": request.actor if request else None, - "menu_links": menu_links, - "display_actor": display_actor, - "show_logout": request is not None - and "ds_actor" in request.cookies - and request.actor, - "zip": zip, - "body_scripts": body_scripts, - "format_bytes": format_bytes, - "show_messages": lambda: self._show_messages(request), - "extra_css_urls": await self._asset_urls( - "extra_css_urls", template, context, request, view_name - ), - "extra_js_urls": await self._asset_urls( - "extra_js_urls", template, context, request, view_name - ), - "base_url": self.setting("base_url"), - "datasette_version": __version__, + **{ + "request": request, + "crumb_items": self._crumb_items, + "urls": self.urls, + "actor": request.actor if request else None, + "menu_links": menu_links, + "display_actor": display_actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, + "zip": zip, + "body_scripts": body_scripts, + "format_bytes": format_bytes, + "show_messages": lambda: self._show_messages(request), + "extra_css_urls": await self._asset_urls( + "extra_css_urls", template, context, request, view_name + ), + "extra_js_urls": await self._asset_urls( + "extra_js_urls", template, context, request, view_name + ), + "base_url": self.setting("base_url"), + "datasette_version": __version__, + }, **extra_template_vars, } if request and request.args.get("_context") and self.setting("template_debug"): @@ -2925,8 +2941,7 @@ class DatasetteRouter: custom_response ), "Default forbidden() hook should have been called" return await custom_response.asgi_send(send) - except Exception as exception: # noqa: BLE001 - # This IS the top-level error handler - it must catch everything + except Exception as exception: return await self.handle_exception(request, send, exception) async def handle_401(self, request, send, exception): @@ -2948,7 +2963,7 @@ class DatasetteRouter: request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") ) if request.query_string: - new_path += f"?{request.query_string}" + new_path += "?{}".format(request.query_string) await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it @@ -3158,7 +3173,8 @@ _curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): # Drop the ".html" suffix - filepath = filepath.removesuffix(".html") + if filepath.endswith(".html"): + filepath = filepath[: -len(".html")] re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index b6c8b77f..4d8c6bea 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -1,8 +1,7 @@ -import hashlib - from datasette import hookimpl +from datasette.utils.asgi import Response, BadRequest from datasette.utils import to_css_class -from datasette.utils.asgi import BadRequest, Response +import hashlib _BLOB_COLUMN = "_blob_column" _BLOB_HASH = "_blob_hash" diff --git a/datasette/cli.py b/datasette/cli.py index 57db83b6..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -1,45 +1,43 @@ import asyncio +import uvicorn +import click +from click import formatting +from click.types import CompositeParamType +from click_default_group import DefaultGroup import functools import json import os import pathlib +from runpy import run_module import shutil +from subprocess import call import sys import textwrap import webbrowser -from runpy import run_module -from subprocess import call - -import click -import uvicorn -from click import formatting -from click.types import CompositeParamType -from click_default_group import DefaultGroup - from .app import ( + Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, - Datasette, pm, ) from .inspect import inspect_tables from .utils import ( - ConnectionProblem, LoadExtension, - SpatialiteConnectionProblem, - SpatialiteNotFound, StartupError, - StaticMount, - ValueAsBooleanError, check_connection, deep_dict_update, find_spatialite, + parse_metadata, + ConnectionProblem, + SpatialiteConnectionProblem, initial_path_for_datasette, pairs_to_nested_config, - parse_metadata, temporary_docker_directory, value_as_boolean, + SpatialiteNotFound, + StaticMount, + ValueAsBooleanError, ) from .utils.sqlite import sqlite3 from .utils.testing import TestClient @@ -77,7 +75,7 @@ class Setting(CompositeParamType): # Datasette 1.0, we turn bare setting names into setting.name # Type checking for those older settings default = DEFAULT_SETTINGS[name] - name = f"settings.{name}" + name = "settings.{}".format(name) if isinstance(default, bool): try: return name, "true" if value_as_boolean(value) else "false" @@ -173,6 +171,7 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" + pass # Register publish plugins @@ -579,27 +578,27 @@ def serve( # https://github.com/simonw/datasette/issues/2389 deep_dict_update(config_data, settings_updates) - kwargs = { - "immutables": immutable, - "cache_headers": not reload, - "cors": cors, - "inspect_data": inspect_data, - "config": config_data, - "metadata": metadata_data, - "sqlite_extensions": sqlite_extensions, - "template_dir": template_dir, - "plugins_dir": plugins_dir, - "static_mounts": static, - "settings": None, # These are passed in config= now - "memory": memory, - "secret": secret, - "version_note": version_note, - "pdb": pdb, - "crossdb": crossdb, - "nolock": nolock, - "internal": internal, - "default_deny": default_deny, - } + kwargs = dict( + immutables=immutable, + cache_headers=not reload, + cors=cors, + inspect_data=inspect_data, + config=config_data, + metadata=metadata_data, + sqlite_extensions=sqlite_extensions, + template_dir=template_dir, + plugins_dir=plugins_dir, + static_mounts=static, + settings=None, # These are passed in config= now + memory=memory, + secret=secret, + version_note=version_note, + pdb=pdb, + crossdb=crossdb, + nolock=nolock, + internal=internal, + default_deny=default_deny, + ) # Separate directories from files directories = [f for f in files if os.path.isdir(f)] @@ -622,7 +621,9 @@ def serve( conn.close() else: raise click.ClickException( - f"Invalid value for '[FILES]...': Path '{file}' does not exist." + "Invalid value for '[FILES]...': Path '{}' does not exist.".format( + file + ) ) # Check for duplicate files by resolving all paths to their absolute forms @@ -683,7 +684,7 @@ def serve( client = TestClient(ds) request_headers = {} if token: - request_headers["Authorization"] = f"Bearer {token}" + request_headers["Authorization"] = "Bearer {}".format(token) cookies = {} if actor: cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) @@ -718,13 +719,9 @@ def serve( path = run_sync(lambda: initial_path_for_datasette(ds)) url = f"http://{host}:{port}{path}" webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } + uvicorn_kwargs = dict( + host=host, port=port, log_level="info", lifespan="on", workers=1 + ) if uds: uvicorn_kwargs["uds"] = uds if ssl_keyfile: @@ -888,7 +885,7 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - f"Connection to {database.path} failed check: {e.args[0]!s}" + f"Connection to {database.path} failed check: {str(e.args[0])}" ) # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning if ( @@ -896,5 +893,9 @@ async def check_databases(ds): and len([db for db in ds.databases.values() if not db.is_memory]) > SQLITE_LIMIT_ATTACHED ): - msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases" + msg = ( + "Warning: --crossdb only works with the first {} attached databases".format( + SQLITE_LIMIT_ATTACHED + ) + ) click.echo(click.style(msg, bold=True, fg="yellow"), err=True) diff --git a/datasette/column_types.py b/datasette/column_types.py index 92fdd969..11a14ec0 100644 --- a/datasette/column_types.py +++ b/datasette/column_types.py @@ -64,14 +64,14 @@ class ColumnType: Return an HTML string to render this cell value, or None to fall through to the default render_cell plugin hook chain. """ - return + return None async def validate(self, value, datasette): """ Validate a value before it is written. Return None if valid, or a string error message if invalid. """ - return + return None async def transform_value(self, value, datasette): """ diff --git a/datasette/csrf.py b/datasette/csrf.py index a62f9473..df239aee 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -40,12 +40,12 @@ def _origin_tuple(value): scheme = (parsed.scheme or "").lower() host = (parsed.hostname or "").lower() if not scheme or not host: - raise ValueError(f"missing scheme or host in {value!r}") + raise ValueError("missing scheme or host in {!r}".format(value)) port = parsed.port # may raise ValueError on bad ports if port is None: port = DEFAULT_PORTS.get(scheme) if port is None: - raise ValueError(f"unknown default port for scheme {scheme!r}") + raise ValueError("unknown default port for scheme {!r}".format(scheme)) return scheme, host, port @@ -125,7 +125,9 @@ class CrossOriginProtectionMiddleware: return await self._forbid( send, - f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'", + "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( + sec_fetch_site + ), ) return @@ -139,11 +141,11 @@ class CrossOriginProtectionMiddleware: request_scheme = self._request_scheme(scope) try: origin_tuple = _origin_tuple(origin) - expected_tuple = _origin_tuple(f"{request_scheme}://{host}") + expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) except ValueError: await self._forbid( send, - f"Malformed Origin {origin!r} or Host {host!r}", + "Malformed Origin {!r} or Host {!r}".format(origin, host), ) return @@ -153,7 +155,7 @@ class CrossOriginProtectionMiddleware: await self._forbid( send, - f"Origin {origin!r} does not match Host {host!r}", + "Origin {!r} does not match Host {!r}".format(origin, host), ) def _request_scheme(self, scope): @@ -161,8 +163,7 @@ class CrossOriginProtectionMiddleware: try: if self.datasette.setting("force_https_urls"): return "https" - except Exception: # noqa: BLE001, S110 - # Settings may not be readable this early; fall back to the ASGI scheme + except Exception: pass return scope.get("scheme") or "http" diff --git a/datasette/database.py b/datasette/database.py index e162d34e..bab3a378 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,18 +1,16 @@ import asyncio import atexit +from collections import namedtuple import inspect import os +from pathlib import Path import queue +import sqlite_utils import sys import tempfile import threading import uuid -from collections import namedtuple -from pathlib import Path -import sqlite_utils - -from .inspect import inspect_hash from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -23,13 +21,14 @@ from .utils import ( get_all_foreign_keys, get_outbound_foreign_keys, md5_not_usedforsecurity, - sqlite3, sqlite_timelimit, - table_column_details, + sqlite3, table_columns, + table_column_details, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_hidden_table_names +from .inspect import inspect_hash connections = threading.local() @@ -100,7 +99,9 @@ class Database: def _check_not_closed(self): if self._closed: - raise DatasetteClosedError(f"Database {self.name!r} has been closed") + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) def _remove_pending_execute_future(self, future): with self._pending_execute_futures_lock: @@ -139,7 +140,7 @@ class Database: if write: extra_kwargs["isolation_level"] = "IMMEDIATE" if self.memory_name: - uri = f"file:{self.memory_name}?mode=memory&cache=shared" + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) conn = sqlite3.connect( uri, uri=True, check_same_thread=False, **extra_kwargs ) @@ -192,20 +193,21 @@ class Database: write_thread.join(timeout=10) if write_thread.is_alive(): sys.stderr.write( - f"Datasette: write thread for {self.name!r} did not exit within 10s\n" + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) ) sys.stderr.flush() for future in pending_execute_futures: try: future.result() - except Exception: # noqa: BLE001, S110 - # Shutdown teardown - a failed pending write must not block close() + except Exception: pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._all_file_connections = [] # Drop per-thread cached read connections we can reach @@ -217,13 +219,13 @@ class Database: if self._read_connection is not None: try: self._read_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._read_connection = None if self._write_connection is not None: try: self._write_connection.close() - except Exception: # noqa: BLE001, S110 + except Exception: pass self._write_connection = None if self.is_temp_disk: @@ -369,8 +371,7 @@ class Database: async def _dispatch_events_after_write(): try: await reply_future - except Exception: # noqa: BLE001 - # The write failed; skip success events regardless of why + except Exception: # if the write failed, don't emit success events return for event in pending_events: @@ -423,7 +424,9 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) - self._write_thread.name = f"_execute_writes for database {self.name}" + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() @@ -444,8 +447,7 @@ class Database: try: conn = self.connect(write=True) self.ds._prepare_connection(conn, self.name) - except Exception as e: # noqa: BLE001 - # Stored and re-raised to whoever queues the next write + except Exception as e: conn_exception = e while True: task = self._write_queue.get() @@ -453,8 +455,7 @@ class Database: if conn is not None: try: conn.close() - except Exception: # noqa: BLE001, S110 - # Best-effort close as the write thread exits + except Exception: pass return exception = None @@ -473,9 +474,8 @@ class Database: except ValueError: # Was probably a memory connection pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e else: @@ -486,8 +486,8 @@ class Database: result = task.fn(conn) else: result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") + except Exception as e: + sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e _deliver_write_result(task, result, exception) @@ -554,7 +554,9 @@ class Database: raise QueryInterrupted(e, sql, params) if log_sql_errors: sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + "ERROR: conn={}, sql = {}, params = {}: {}\n".format( + conn, repr(sql), params, e + ) ) sys.stderr.flush() raise @@ -711,9 +713,9 @@ class Database: column_names and len(column_names) == 2 and ("id" in column_names or "pk" in column_names) - and set(column_names) != {"id", "pk"} + and not set(column_names) == {"id", "pk"} ): - return next(c for c in column_names if c not in ("id", "pk")) + return [c for c in column_names if c not in ("id", "pk")][0] # Couldn't find a label: return None @@ -855,10 +857,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( "fn", - "isolated_connection", + "task_id", "loop", "reply_future", - "task_id", + "isolated_connection", "transaction", ) @@ -899,7 +901,7 @@ class QueryInterrupted(Exception): self.params = params def __str__(self): - return f"QueryInterrupted: {self.e}" + return "QueryInterrupted: {}".format(self.e) class MultipleValues(Exception): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index ee165ae5..602e0df4 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -2,8 +2,8 @@ from datasette import hookimpl from datasette.permissions import Action from datasette.resources import ( DatabaseResource, - QueryResource, TableResource, + QueryResource, ) diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index bff5f0a7..91c1c5aa 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,9 +1,8 @@ +from datasette import hookimpl import datetime import os import time -from datasette import hookimpl - def header(key, request): key = key.replace("_", "-").encode("utf-8") diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index dee5df42..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,29 +17,18 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from .config import config_permissions_sql as config_permissions_sql -from .defaults import ( - DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, -) -from .defaults import ( - default_action_permissions_sql as default_action_permissions_sql, -) -from .defaults import ( - # Avoid "datasette.default_permissions" does not explicitly export attribute - default_allow_sql_check as default_allow_sql_check, -) -from .defaults import ( - default_query_permissions_sql as default_query_permissions_sql, -) -from .restrictions import ( - ActorRestrictions as ActorRestrictions, -) - # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, -) -from .restrictions import ( restrictions_allow_action as restrictions_allow_action, + ActorRestrictions as ActorRestrictions, ) from .root import root_user_permissions_sql as root_user_permissions_sql +from .config import config_permissions_sql as config_permissions_sql +from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute + default_allow_sql_check as default_allow_sql_check, + default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, + DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, +) diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index 4494f07f..8edc976e 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration. from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -55,8 +55,8 @@ class ConfigPermissionProcessor: def __init__( self, - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, ): self.datasette = datasette @@ -74,8 +74,8 @@ class ConfigPermissionProcessor: self.restrictions = actor.get("_r", {}) if actor else {} # Pre-compute restriction info for efficiency - self.restricted_databases: set[str] = set() - self.restricted_tables: set[tuple[str, str]] = set() + self.restricted_databases: Set[str] = set() + self.restricted_tables: Set[Tuple[str, str]] = set() if self.has_restrictions: self.restricted_databases = { @@ -92,7 +92,7 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - def evaluate_allow_block(self, allow_block: Any) -> bool | None: + def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]: """Evaluate an allow block against the current actor.""" if allow_block is None: return None @@ -104,8 +104,8 @@ class ConfigPermissionProcessor: def is_in_restriction_allowlist( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], ) -> bool: """Check if resource is allowed by actor restrictions.""" if not self.has_restrictions: @@ -147,9 +147,9 @@ class ConfigPermissionProcessor: def add_permissions_rule( self, - parent: str | None, - child: str | None, - permissions_block: dict | None, + parent: Optional[str], + child: Optional[str], + permissions_block: Optional[dict], scope_desc: str, ) -> None: """Add a rule from a permissions:{action} block.""" @@ -169,8 +169,8 @@ class ConfigPermissionProcessor: def add_allow_block_rule( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], allow_block: Any, scope_desc: str, ) -> None: @@ -202,8 +202,8 @@ class ConfigPermissionProcessor: def _add_restriction_gate_denies( self, - parent: str | None, - child: str | None, + parent: Optional[str], + child: Optional[str], is_allowed: bool, scope_desc: str, ) -> None: @@ -235,7 +235,7 @@ class ConfigPermissionProcessor: if db_name == parent: self.collector.add(db_name, table_name, False, reason) - def process(self) -> PermissionSQL | None: + def process(self) -> Optional[PermissionSQL]: """Process all config rules and return combined PermissionSQL.""" self._process_root_permissions() self._process_databases() @@ -425,10 +425,10 @@ class ConfigPermissionProcessor: @hookimpl(specname="permission_resources_sql") async def config_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Apply permission rules from datasette.yaml configuration. diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 6f97812b..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -29,28 +29,29 @@ DEFAULT_ALLOW_ACTIONS = frozenset( @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Enforce the default_allow_sql setting. When default_allow_sql is false (the default), execute-sql is denied unless explicitly allowed by config or other rules. """ - if action == "execute-sql" and not datasette.setting("default_allow_sql"): - return PermissionSQL.deny(reason="default_allow_sql is false") + if action == "execute-sql": + if not datasette.setting("default_allow_sql"): + return PermissionSQL.deny(reason="default_allow_sql is false") return None @hookimpl(specname="permission_resources_sql") async def default_action_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: """ Provide default allow rules for standard view/execute actions. @@ -70,10 +71,10 @@ async def default_action_permissions_sql( @hookimpl(specname="permission_resources_sql") async def default_query_permissions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> PermissionSQL | None: +) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None if action not in {"view-query", "update-query", "delete-query"}: diff --git a/datasette/default_permissions/helpers.py b/datasette/default_permissions/helpers.py index 5e59b7b4..47e03569 100644 --- a/datasette/default_permissions/helpers.py +++ b/datasette/default_permissions/helpers.py @@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set if TYPE_CHECKING: from datasette.app import Datasette @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datasette.permissions import PermissionSQL -def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: +def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]: """ Get all name variants for an action (full name and abbreviation). @@ -27,7 +27,7 @@ def get_action_name_variants(datasette: Datasette, action: str) -> set[str]: return variants -def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool: +def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool: """Check if an action (or its abbreviation) is in a list.""" return bool(get_action_name_variants(datasette, action).intersection(action_list)) @@ -36,8 +36,8 @@ def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool class PermissionRow: """A single permission rule row.""" - parent: str | None - child: str | None + parent: Optional[str] + child: Optional[str] allow: bool reason: str @@ -46,14 +46,14 @@ class PermissionRowCollector: """Collects permission rows and converts them to PermissionSQL.""" def __init__(self, prefix: str = "row"): - self.rows: list[PermissionRow] = [] + self.rows: List[PermissionRow] = [] self.prefix = prefix def add( self, - parent: str | None, - child: str | None, - allow: bool | None, + parent: Optional[str], + child: Optional[str], + allow: Optional[bool], reason: str, if_not_none: bool = False, ) -> None: @@ -62,7 +62,7 @@ class PermissionRowCollector: return self.rows.append(PermissionRow(parent, child, allow, reason)) - def to_permission_sql(self) -> PermissionSQL | None: + def to_permission_sql(self) -> Optional[PermissionSQL]: """Convert collected rows to a PermissionSQL object.""" if not self.rows: return None diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index 88e1d274..a22cd7e5 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -8,7 +8,7 @@ contains allowlists of resources the actor can access. from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Set, Tuple if TYPE_CHECKING: from datasette.app import Datasette @@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants class ActorRestrictions: """Parsed actor restrictions from the _r key.""" - global_actions: list[str] # _r.a - globally allowed actions + global_actions: List[str] # _r.a - globally allowed actions database_actions: dict # _r.d - {db_name: [actions]} table_actions: dict # _r.r - {db_name: {table: [actions]}} @classmethod - def from_actor(cls, actor: dict | None) -> ActorRestrictions | None: + def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]: """Parse restrictions from actor dict. Returns None if no restrictions.""" if not actor: return None @@ -44,11 +44,11 @@ class ActorRestrictions: table_actions=restrictions.get("r", {}), ) - def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool: + def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool: """Check if action is in the global allowlist.""" return action_in_list(datasette, action, self.global_actions) - def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]: + def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]: """Get database names where this action is allowed.""" allowed = set() for db_name, db_actions in self.database_actions.items(): @@ -57,8 +57,8 @@ class ActorRestrictions: return allowed def get_allowed_tables( - self, datasette: Datasette, action: str - ) -> set[tuple[str, str]]: + self, datasette: "Datasette", action: str + ) -> Set[Tuple[str, str]]: """Get (database, table) pairs where this action is allowed.""" allowed = set() for db_name, tables in self.table_actions.items(): @@ -70,10 +70,10 @@ class ActorRestrictions: @hookimpl(specname="permission_resources_sql") async def actor_restrictions_sql( - datasette: Datasette, - actor: dict | None, + datasette: "Datasette", + actor: Optional[dict], action: str, -) -> list[PermissionSQL] | None: +) -> Optional[List[PermissionSQL]]: """ Handle actor restriction-based permission rules. @@ -140,10 +140,10 @@ async def actor_restrictions_sql( def restrictions_allow_action( - datasette: Datasette, + datasette: "Datasette", restrictions: dict, action: str, - resource: str | tuple[str, str] | None, + resource: Optional[str | Tuple[str, str]], ) -> bool: """ Check if restrictions allow the requested action on the requested resource. diff --git a/datasette/default_permissions/root.py b/datasette/default_permissions/root.py index 22d13f65..4931f7ff 100644 --- a/datasette/default_permissions/root.py +++ b/datasette/default_permissions/root.py @@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL @hookimpl(specname="permission_resources_sql") async def root_user_permissions_sql( - datasette: Datasette, - actor: dict | None, -) -> PermissionSQL | None: + datasette: "Datasette", + actor: Optional[dict], +) -> Optional[PermissionSQL]: """ Grant root user full permissions when --root flag is used. """ diff --git a/datasette/default_permissions/tokens.py b/datasette/default_permissions/tokens.py index 52daf8a2..7a359dc6 100644 --- a/datasette/default_permissions/tokens.py +++ b/datasette/default_permissions/tokens.py @@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from datasette.app import Datasette @@ -17,13 +17,15 @@ from datasette.tokens import SignedTokenHandler @hookimpl -def register_token_handler(datasette: Datasette): +def register_token_handler(datasette: "Datasette"): """Register the default signed token handler.""" return SignedTokenHandler() @hookimpl(specname="actor_from_request") -async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None: +async def actor_from_signed_api_token( + datasette: "Datasette", request +) -> Optional[dict]: """ Authenticate requests using API tokens by delegating to all registered token handlers via datasette.verify_token(). diff --git a/datasette/default_table_actions.py b/datasette/default_table_actions.py index 0f2f32ef..e41434ef 100644 --- a/datasette/default_table_actions.py +++ b/datasette/default_table_actions.py @@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request): "label": "Alter table", "description": "Change columns and primary key for this table.", "attrs": { - "aria-label": f"Alter table {table}", + "aria-label": "Alter table {}".format(table), "data-table-action": "alter-table", }, } diff --git a/datasette/events.py b/datasette/events.py index 5f3fd06e..e8786da9 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -1,8 +1,7 @@ from abc import ABC, abstractproperty from dataclasses import asdict, dataclass, field -from datetime import datetime, timezone - from datasette.hookspecs import hookimpl +from datetime import datetime, timezone @dataclass diff --git a/datasette/facets.py b/datasette/facets.py index 8c09e1dc..5f757df3 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,13 +1,12 @@ import json import urllib - from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( - detect_json1, escape_sqlite, path_with_added_args, path_with_removed_args, + detect_json1, sqlite3, ) @@ -31,7 +30,7 @@ def load_facet_configs(request, table_config): assert ( len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, facet_config = next(iter(facet_config.items())) + type, facet_config = list(facet_config.items())[0] if isinstance(facet_config, str): facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( @@ -161,13 +160,18 @@ class ColumnFacet(Facet): for column in columns: if column in already_enabled: continue - suggested_facet_sql = f""" - with limited as (select * from ({self.sql}) limit {self.suggest_consider}) - select {escape_sqlite(column)} as value, count(*) as n from limited + suggested_facet_sql = """ + with limited as (select * from ({sql}) limit {suggest_consider}) + select {column} as value, count(*) as n from limited where value is not null group by value - limit {facet_size + 1} - """ + limit {limit} + """.format( + column=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + suggest_consider=self.suggest_consider, + ) distinct_values = None try: distinct_values = await self.ds.execute( @@ -263,7 +267,7 @@ class ColumnFacet(Facet): for row in facet_rows: column_qs = column if column.startswith("_"): - column_qs = f"{column}__exact" + column_qs = "{}__exact".format(column) selected = (column_qs, str(row["value"])) in qs_pairs if selected: toggle_path = path_with_removed_args( @@ -338,12 +342,12 @@ class ArrayFacet(Facet): for v in await self.ds.execute( self.database, ( - f"select {escape_sqlite(column)} from ({self.sql}) " - f"where {escape_sqlite(column)} is not null " - f"and {escape_sqlite(column)} != '' " - f"and json_array_length({escape_sqlite(column)}) > 0 " + "select {column} from ({sql}) " + "where {column} is not null " + "and {column} != '' " + "and json_array_length({column}) > 0 " "limit 100" - ), + ).format(column=escape_sqlite(column), sql=self.sql), self.params, truncate=False, custom_time_limit=self.ds.setting( @@ -384,14 +388,14 @@ class ArrayFacet(Facet): source = source_and_config["source"] column = config.get("column") or config["simple"] # https://github.com/simonw/datasette/issues/448 - facet_sql = f""" - with inner as ({self.sql}), + facet_sql = """ + with inner as ({sql}), deduped_array_items as ( select distinct j.value, inner.* from - json_each([inner].{escape_sqlite(column)}) j + json_each([inner].{col}) j join inner ) select @@ -402,8 +406,12 @@ class ArrayFacet(Facet): group by value order by - count(*) desc, value limit {facet_size + 1} - """ + count(*) desc, value limit {limit} + """.format( + col=escape_sqlite(column), + sql=self.sql, + limit=facet_size + 1, + ) try: facet_rows_results = await self.ds.execute( self.database, diff --git a/datasette/filters.py b/datasette/filters.py index 1d4e32c2..95cc5f37 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,11 +1,8 @@ -import json -from typing import ClassVar - from datasette import hookimpl from datasette.resources import DatabaseResource -from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError - +from datasette.utils.asgi import BadRequest +import json from .utils import detect_json1, escape_sqlite, path_with_removed_args @@ -102,9 +99,9 @@ def search_filters(request, database, table, datasette): fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), match_clause=( - f":search_{i}" + ":search_{}".format(i) if search_mode_raw - else f"escape_fts(:search_{i})" + else "escape_fts(:search_{})".format(i) ), ) ) @@ -137,11 +134,11 @@ def through_filters(request, database, table, datasette): value = through_data["value"] db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - fk_to_us = next( - (fk for fk in outgoing_foreign_keys if fk["other_table"] == table), - None, - ) - if fk_to_us is None: + try: + fk_to_us = [ + fk for fk in outgoing_foreign_keys if fk["other_table"] == table + ][0] + except IndexError: raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) @@ -368,7 +365,7 @@ class Filters: ), ] ) - _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters} + _filters_by_key = {f.key: f for f in _filters} def __init__(self, pairs): self.pairs = pairs diff --git a/datasette/fixtures.py b/datasette/fixtures.py index 049e35ed..7c85e16a 100644 --- a/datasette/fixtures.py +++ b/datasette/fixtures.py @@ -1,10 +1,9 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented import itertools import random import string -from datasette.utils import documented -from datasette.utils.sqlite import sqlite3 - __all__ = [ "EXTRA_DATABASE_SQL", "TABLES", @@ -347,7 +346,9 @@ CREATE VIEW searchable_view_configured_by_metadata AS + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + "\n".join( [ - f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");' + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) for a, b, c, content in generate_compound_rows(1001) ] ) diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 67bf0d8b..91b1ff96 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,5 +1,4 @@ -from datasette import Response, hookimpl - +from datasette import hookimpl, Response from .utils import add_cors_headers diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index c36d5dbe..e255ddf2 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,21 +1,16 @@ -import traceback - -from markupsafe import Markup - -from datasette import Response, hookimpl - +from datasette import hookimpl, Response from .utils import add_cors_headers, error_body from .utils.asgi import ( Base400, ) from .views.base import DatasetteError +from markupsafe import Markup +import traceback -# Debugger imports are deliberate - they back the "pdb" setting, which drops -# into a debugger on unhandled exceptions try: - import ipdb as pdb # noqa: T100 + import ipdb as pdb except ImportError: - import pdb # noqa: T100 + import pdb try: import rich @@ -74,7 +69,7 @@ def handle_exception(datasette, request, exception): dict( info, urls=datasette.urls, - menu_links=list, + menu_links=lambda: [], ) ), status=status, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f89f2f36..7c56f882 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -1,4 +1,5 @@ -from pluggy import HookimplMarker, HookspecMarker +from pluggy import HookimplMarker +from pluggy import HookspecMarker hookspec = HookspecMarker("datasette") hookimpl = HookimplMarker("datasette") diff --git a/datasette/inspect.py b/datasette/inspect.py index b126ce5c..5e681e03 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -1,13 +1,13 @@ import hashlib from .utils import ( + detect_spatialite, detect_fts, detect_primary_keys, - detect_spatialite, escape_sqlite, get_all_foreign_keys, - sqlite3, table_columns, + sqlite3, ) HASH_BLOCK_SIZE = 1024 * 1024 @@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata): """) ] - for t, table_info in tables.items(): + for t in tables.keys(): for hidden_table in hidden_tables: if t == hidden_table or t.startswith(hidden_table): - table_info["hidden"] = True + tables[t]["hidden"] = True continue return tables diff --git a/datasette/jump.py b/datasette/jump.py index d70d33df..d138e827 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -21,7 +21,7 @@ class JumpSQL: search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", - ) -> JumpSQL: + ) -> "JumpSQL": if search_text is None: search_text = " ".join( text for text in (label, display_name, description) if text is not None diff --git a/datasette/permissions.py b/datasette/permissions.py index e03b065c..786dc026 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,7 +1,7 @@ -import contextvars from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple +import contextvars # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( @@ -72,8 +72,8 @@ class Resource(ABC): ) def __repr__(self) -> str: - return ( - f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})" + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child ) @property @@ -129,6 +129,7 @@ class Resource(ABC): Must return two columns: parent, child """ + pass class AllowedResource(NamedTuple): diff --git a/datasette/plugins.py b/datasette/plugins.py index 9cf94079..ae2cb17d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,14 +1,20 @@ import importlib -import importlib.metadata as importlib_metadata -import importlib.resources as importlib_resources import os -import sys -from pprint import pprint - import pluggy - +from pprint import pprint +import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: + import importlib_metadata + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -79,7 +85,7 @@ if DATASETTE_LOAD_PLUGINS is not None: # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) except importlib_metadata.PackageNotFoundError: - sys.stderr.write(f"Plugin {package_name} could not be found\n") + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) # Load default plugins diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 9ace865b..63d22fe8 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -1,17 +1,15 @@ +from datasette import hookimpl +import click import json import os import re from subprocess import CalledProcessError, check_call, check_output -import click - -from datasette import hookimpl - -from ..utils import temporary_docker_directory from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from ..utils import temporary_docker_directory @hookimpl @@ -221,7 +219,7 @@ def publish_subcommand(publish): check_call( "gcloud builds submit --tag {}{}".format( - image_id, f" --timeout {timeout}" if timeout else "" + image_id, " --timeout {}".format(timeout) if timeout else "" ), shell=True, ) @@ -233,7 +231,7 @@ def publish_subcommand(publish): ("--min-instances", min_instances), ): if value is not None: - extra_deploy_options.append(f"{option} {value}") + extra_deploy_options.append("{} {}".format(option, value)) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, @@ -260,16 +258,24 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi ) from exc describe_cmd = ( - f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} " - f"--location {artifact_region} --quiet" + "gcloud artifacts repositories describe {repo} --project {project} " + "--location {location} --quiet" + ).format( + repo=artifact_repository, + project=artifact_project, + location=artifact_region, ) try: check_call(describe_cmd, shell=True) return except CalledProcessError: create_cmd = ( - f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker " - f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet' + "gcloud artifacts repositories create {repo} --repository-format=docker " + '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet' + ).format( + repo=artifact_repository, + location=artifact_region, + project=artifact_project, ) try: check_call(create_cmd, shell=True) diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 27dfd4bf..29665eb3 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -1,11 +1,9 @@ +from ..utils import StaticMount +import click import os import shutil import sys -import click - -from ..utils import StaticMount - def add_common_publish_arguments_and_options(subcommand): for decorator in reversed( @@ -78,7 +76,9 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): """Exit (with error message) if ``binary` isn't installed""" if not shutil.which(binary): click.secho( - f"Publishing to {publish_target} requires {binary} to be installed and configured", + "Publishing to {publish_target} requires {binary} to be installed and configured".format( + publish_target=publish_target, binary=binary + ), bg="red", fg="white", bold=True, diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index b0290833..f576a346 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -1,21 +1,19 @@ +from contextlib import contextmanager +from datasette import hookimpl +import click import json import os import pathlib import shlex import shutil -import tempfile -from contextlib import contextmanager from subprocess import call, check_output - -import click - -from datasette import hookimpl -from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata +import tempfile from .common import ( add_common_publish_arguments_and_options, fail_if_publish_binary_not_installed, ) +from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata @hookimpl @@ -236,7 +234,7 @@ def temporary_heroku_directory( extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( - [f"-i {shlex.quote(file_name)}" for file_name in file_names] + ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] ) procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0e01f52f..7c94f6ee 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,13 +1,12 @@ import json - from datasette.extras import extra_names_from_request from datasette.utils import ( - CustomJSONEncoder, error_body, - path_from_row_pks, - remove_infinites, - sqlite3, value_as_boolean, + remove_infinites, + CustomJSONEncoder, + path_from_row_pks, + sqlite3, ) from datasette.utils.asgi import Response diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index db3c6548..f5f977d9 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,9 +1,8 @@ from __future__ import annotations -import json -from collections.abc import Iterable from dataclasses import dataclass -from typing import Any +import json +from typing import Any, Iterable from .utils import tilde_encode, urlsafe_components @@ -387,7 +386,7 @@ async def count_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) @@ -463,7 +462,7 @@ async def list_queries( except ValueError: components = [] if database is None and len(components) == 3: - where_clauses.append(f""" + where_clauses.append(""" ( q.database_name > :cursor_database OR ( @@ -477,12 +476,12 @@ async def list_queries( ) ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_database"] = components[0] params["cursor_sort_key"] = components[1] params["cursor_name"] = components[2] elif database is not None and len(components) == 2: - where_clauses.append(f""" + where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key OR ( @@ -490,7 +489,7 @@ async def list_queries( AND q.name > :cursor_name ) ) - """) + """.format(sort_key_sql=sort_key_sql)) params["cursor_sort_key"] = components[0] params["cursor_name"] = components[1] @@ -503,7 +502,7 @@ async def list_queries( OR q.sql LIKE :query_search ) """) - params["query_search"] = f"%{q}%" + params["query_search"] = "%{}%".format(q) if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) diff --git a/datasette/tokens.py b/datasette/tokens.py index 79f840d2..4f905339 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -10,7 +10,7 @@ from __future__ import annotations import dataclasses import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import itsdangerous @@ -50,24 +50,24 @@ class TokenRestrictions: database: dict[str, list[str]] = dataclasses.field(default_factory=dict) resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict) - def allow_all(self, action: str) -> TokenRestrictions: + def allow_all(self, action: str) -> "TokenRestrictions": """Allow an action across all databases and resources.""" self.all.append(action) return self - def allow_database(self, database: str, action: str) -> TokenRestrictions: + def allow_database(self, database: str, action: str) -> "TokenRestrictions": """Allow an action on a specific database.""" self.database.setdefault(database, []).append(action) return self def allow_resource( self, database: str, resource: str, action: str - ) -> TokenRestrictions: + ) -> "TokenRestrictions": """Allow an action on a specific resource within a database.""" self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self - def abbreviated(self, datasette: Datasette) -> dict | None: + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: """ Return the abbreviated ``_r`` dictionary shape for this set of restrictions, using action abbreviations registered with ``datasette``. @@ -112,16 +112,16 @@ class TokenHandler: async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: """Create and return a token string for the given actor.""" raise NotImplementedError - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: """ Verify a token and return an actor dict. @@ -142,11 +142,11 @@ class SignedTokenHandler(TokenHandler): async def create_token( self, - datasette: Datasette, + datasette: "Datasette", actor_id: str, *, - expires_after: int | None = None, - restrictions: TokenRestrictions | None = None, + expires_after: Optional[int] = None, + restrictions: Optional[TokenRestrictions] = None, ) -> str: if not datasette.setting("allow_signed_tokens"): raise ValueError( @@ -163,7 +163,7 @@ class SignedTokenHandler(TokenHandler): token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) - async def verify_token(self, datasette: Datasette, token: str) -> dict | None: + async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: prefix = "dstok_" if not token.startswith(prefix): @@ -200,8 +200,9 @@ class SignedTokenHandler(TokenHandler): ): duration = max_signed_tokens_ttl - if duration and time.time() - created > duration: - raise TokenInvalid("Token has expired") + if duration: + if time.time() - created > duration: + raise TokenInvalid("Token has expired") actor = {"id": decoded["a"], "token": "dstok"} diff --git a/datasette/tracer.py b/datasette/tracer.py index 1fbda6f9..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,11 +1,10 @@ import asyncio -import json -import time -import traceback from contextlib import contextmanager from contextvars import ContextVar - from markupsafe import escape +import time +import json +import traceback tracers = {} @@ -133,17 +132,17 @@ class AsgiTracer: "num_traces": len(traces), "traces": traces, } - content_type = next( - ( + try: + content_type = [ v.decode("utf8") for k, v in response_headers if k.lower() == b"content-type" - ), - "", - ) + ][0] + except IndexError: + content_type = "" if "text/html" in content_type and b"" in accumulated_body: extra = escape(json.dumps(trace_info, indent=2)) - extra_html = f"
{extra}
".encode() + extra_html = f"
{extra}
".encode("utf8") accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index f8da20f3..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,7 +1,6 @@ +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib -from .utils import PrefixedUrlString, path_with_format, tilde_encode - class Urls: def __init__(self, ds): @@ -9,7 +8,8 @@ class Urls: def path(self, path, format=None): if not isinstance(path, PrefixedUrlString): - path = path.removeprefix("/") + if path.startswith("/"): + path = path[1:] path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) @@ -56,7 +56,6 @@ class Urls: return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): - return ( - self.table(database, table) - + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}" + return self.table(database, table) + "/{}.blob?_blob_column={}".format( + row_path, urllib.parse.quote_plus(column) ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index eb46d549..18d3ba52 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,31 +1,29 @@ import asyncio -import base64 import binascii +from contextlib import contextmanager +import aiofiles +import click +from collections import OrderedDict, namedtuple, Counter import copy import dataclasses +import base64 import hashlib import inspect import json -import os -import re -import secrets -import shlex -import shutil -import tempfile -import time -import types -import typing -import urllib -from collections import Counter, OrderedDict, namedtuple -from collections.abc import Iterable -from contextlib import contextmanager - -import aiofiles -import click import markupsafe import mergedeep +import os +import re +import shlex +import tempfile +import typing +import time +import types +import secrets +import shutil +from typing import Iterable, List, Tuple +import urllib import yaml - from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo @@ -38,7 +36,7 @@ if typing.TYPE_CHECKING: class PaginatedResources: """Paginated results from allowed_resources query.""" - resources: list["Resource"] + resources: List["Resource"] next: str | None # Keyset token for next page (None if no more results) _datasette: typing.Any = dataclasses.field(default=None, repr=False) _action: str = dataclasses.field(default=None, repr=False) @@ -85,132 +83,22 @@ class PaginatedResources: # From https://www.sqlite.org/lang_keywords.html -reserved_words = { - "abort", - "action", - "add", - "after", - "all", - "alter", - "analyze", - "and", - "as", - "asc", - "attach", - "autoincrement", - "before", - "begin", - "between", - "by", - "cascade", - "case", - "cast", - "check", - "collate", - "column", - "commit", - "conflict", - "constraint", - "create", - "cross", - "current_date", - "current_time", - "current_timestamp", - "database", - "default", - "deferrable", - "deferred", - "delete", - "desc", - "detach", - "distinct", - "drop", - "each", - "else", - "end", - "escape", - "except", - "exclusive", - "exists", - "explain", - "fail", - "for", - "foreign", - "from", - "full", - "glob", - "group", - "having", - "if", - "ignore", - "immediate", - "in", - "index", - "indexed", - "initially", - "inner", - "insert", - "instead", - "intersect", - "into", - "is", - "isnull", - "join", - "key", - "left", - "like", - "limit", - "match", - "natural", - "no", - "not", - "notnull", - "null", - "of", - "offset", - "on", - "or", - "order", - "outer", - "plan", - "pragma", - "primary", - "query", - "raise", - "recursive", - "references", - "regexp", - "reindex", - "release", - "rename", - "replace", - "restrict", - "right", - "rollback", - "row", - "savepoint", - "select", - "set", - "table", - "temp", - "temporary", - "then", - "to", - "transaction", - "trigger", - "union", - "unique", - "update", - "using", - "vacuum", - "values", - "view", - "virtual", - "when", - "where", - "with", - "without", -} +reserved_words = set( + ( + "abort action add after all alter analyze and as asc attach autoincrement " + "before begin between by cascade case cast check collate column commit " + "conflict constraint create cross current_date current_time " + "current_timestamp database default deferrable deferred delete desc detach " + "distinct drop each else end escape except exclusive exists explain fail " + "for foreign from full glob group having if ignore immediate in index " + "indexed initially inner insert instead intersect into is isnull join key " + "left like limit match natural no not notnull null of offset on or order " + "outer plan pragma primary query raise recursive references regexp reindex " + "release rename replace restrict right rollback row savepoint select set " + "table temp temporary then to transaction trigger union unique update using " + "vacuum values view virtual when where with without" + ).split() +) APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ @@ -270,7 +158,7 @@ functions_marked_as_documented = [] def documented(fn=None, *, label=None): def decorate(fn): - fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}" + fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) functions_marked_as_documented.append(fn) return fn @@ -472,7 +360,7 @@ disallawed_sql_res = [ ( re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( - ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) + ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) ), ) ] @@ -646,7 +534,10 @@ CMD {cmd}""".format( else "" ), environment_variables="\n".join( - [f"ENV {key} '{value}'" for key, value in environment_variables.items()] + [ + "ENV {} '{}'".format(key, value) + for key, value in environment_variables.items() + ] ), install_from=" ".join(install), files=" ".join(files), @@ -749,7 +640,7 @@ def get_outbound_foreign_keys(conn, table): fks = [] for info in infos: if info is not None: - id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info + id, seq, table_name, from_, to_, on_update, on_delete, match = info fks.append( { "column": from_, @@ -850,7 +741,7 @@ def detect_json1(conn=None): try: conn.execute("SELECT json('{}')") return True - except sqlite3.Error: + except Exception: return False finally: if close_conn: @@ -930,7 +821,9 @@ def is_url(value): if not value.startswith("http://") and not value.startswith("https://"): return False # Any whitespace at all is invalid - return not whitespace_re.search(value) + if whitespace_re.search(value): + return False + return True css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") @@ -983,9 +876,7 @@ def module_from_path(path, name): mod.__file__ = path with open(path, "r") as file: code = compile(file.read(), path, "exec", dont_inherit=True) - # Executing the file is the whole point - this is how --plugins-dir loads - # plugins and how metadata/config .py files are evaluated - exec(code, mod.__dict__) # noqa: S102 + exec(code, mod.__dict__) return mod @@ -1142,7 +1033,9 @@ def escape_fts(query): query += '"' bits = _escape_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) + return " ".join( + '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits + ) class MultiParams: @@ -1154,7 +1047,7 @@ class MultiParams: data[key], (list, tuple) ), "dictionary data should be a dictionary of key => [list]" self._data = data - elif isinstance(data, (list, tuple)): + elif isinstance(data, list) or isinstance(data, tuple): new_data = {} for item in data: assert ( @@ -1244,7 +1137,9 @@ def _gather_arguments(fn, kwargs): for parameter in parameters: if parameter not in kwargs: raise TypeError( - f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}" + "{} requires parameters {}, missing: {}".format( + fn, tuple(parameters), set(parameters) - set(kwargs.keys()) + ) ) call_with.append(kwargs[parameter]) return call_with @@ -1313,9 +1208,9 @@ def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): if list(config.keys()) == ["$env"]: - return environ.get(next(iter(config.values()))) + return environ.get(list(config.values())[0]) elif list(config.keys()) == ["$file"]: - with open(next(iter(config.values()))) as fp: + with open(list(config.values())[0]) as fp: return fp.read() else: return { @@ -1411,7 +1306,7 @@ _named_param_re = re.compile(r":(\w+)") @documented -def named_parameters(sql: str) -> list[str]: +def named_parameters(sql: str) -> List[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement @@ -1424,7 +1319,7 @@ def named_parameters(sql: str) -> list[str]: return _named_param_re.findall(sql) -async def derive_named_parameters(db: "Database", sql: str) -> list[str]: +async def derive_named_parameters(db: "Database", sql: str) -> List[str]: """ This undocumented but stable method exists for backwards compatibility with plugins that were using it before it switched to named_parameters() @@ -1448,9 +1343,9 @@ def parse_size_limit(value, default, maximum, name="_size"): if size < 0: raise ValueError except ValueError: - raise ValueError(f"{name} must be a positive integer") + raise ValueError("{} must be a positive integer".format(name)) if size > maximum: - raise ValueError(f"{name} must be <= {maximum}") + raise ValueError("{} must be <= {}".format(name, maximum)) return size @@ -1508,7 +1403,7 @@ class TildeEncoder(dict): elif b == _space: res = "+" else: - res = f"~{b:02X}" + res = "~{:02X}".format(b) self[b] = res return res @@ -1603,7 +1498,7 @@ def _combine(base: dict, update: dict) -> dict: return base -def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict: +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: """ Parse a list of key-value pairs into a nested dictionary. """ @@ -1618,7 +1513,7 @@ def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm method = getattr(pm.hook, name, None) - assert method is not None, f"No hook found for {name}" + assert method is not None, "No hook found for {}".format(name) async def inner(): html_bits = [] @@ -1642,7 +1537,7 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' and 'allow' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 812194fd..610b86f2 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,29 +1,28 @@ import json -import re -from http.cookies import Morsel, SimpleCookie -from mimetypes import guess_type -from pathlib import Path -from urllib.parse import parse_qs, parse_qsl, urlunparse - -import aiofiles -import aiofiles.os - +from typing import Optional from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file from datasette.utils.multipart import ( - DEFAULT_MAX_FIELD_SIZE, - DEFAULT_MAX_FIELDS, + parse_form_data, + MultipartParseError, + FormData, DEFAULT_MAX_FILE_SIZE, + DEFAULT_MAX_REQUEST_SIZE, + DEFAULT_MAX_FIELDS, DEFAULT_MAX_FILES, + DEFAULT_MAX_PARTS, + DEFAULT_MAX_FIELD_SIZE, DEFAULT_MAX_MEMORY_FILE_SIZE, DEFAULT_MAX_PART_HEADER_BYTES, DEFAULT_MAX_PART_HEADER_LINES, - DEFAULT_MAX_PARTS, - DEFAULT_MAX_REQUEST_SIZE, DEFAULT_MIN_FREE_DISK_BYTES, - FormData, - MultipartParseError, - parse_form_data, ) +from mimetypes import guess_type +from urllib.parse import parse_qs, urlunparse, parse_qsl +from pathlib import Path +from http.cookies import SimpleCookie, Morsel +import aiofiles +import aiofiles.os +import re # Workaround for adding samesite support to pre 3.8 python Morsel._reserved["samesite"] = "SameSite" @@ -89,7 +88,7 @@ class Request: self.max_post_body_bytes = max_post_body_bytes def __repr__(self): - return f'' + return ''.format(self.method, self.url) @property def method(self): @@ -168,7 +167,7 @@ class Request: if max_bytes is None: max_bytes = self.max_post_body_bytes too_large = PayloadTooLarge( - f"Request body exceeded maximum size of {max_bytes} bytes" + "Request body exceeded maximum size of {} bytes".format(max_bytes) ) if max_bytes: # Reject early if the client declares an oversized body @@ -207,7 +206,7 @@ class Request: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -530,9 +529,9 @@ class Response: httponly=False, samesite="lax", ): - assert ( - samesite in SAMESITE_VALUES - ), f"samesite should be one of {SAMESITE_VALUES}" + assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format( + SAMESITE_VALUES + ) cookie = SimpleCookie() cookie[key] = value for prop_name, prop_value in ( diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index 0469d7a8..c4b64908 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/ """ -class BaseConverter: +class BaseConverter(object): decimal_digits = "0123456789" def __init__(self, digits): diff --git a/datasette/utils/check_callable.py b/datasette/utils/check_callable.py index e21a769b..a0997d20 100644 --- a/datasette/utils/check_callable.py +++ b/datasette/utils/check_callable.py @@ -1,6 +1,6 @@ import inspect import types -from typing import Any, NamedTuple +from typing import NamedTuple, Any class CallableStatus(NamedTuple): @@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus: if isinstance(obj, types.FunctionType): return CallableStatus(True, inspect.iscoroutinefunction(obj)) - if callable(obj): + if hasattr(obj, "__call__"): return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__)) - assert False, f"obj {obj!r} is somehow callable with no __call__ method" + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 0ddeb847..702b53d8 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -207,8 +207,7 @@ async def populate_schema_tables(internal_db, db, schema_version): columns = table_column_details(conn, table_name) columns_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } for column in columns @@ -218,8 +217,7 @@ async def populate_schema_tables(internal_db, db, schema_version): ).fetchall() foreign_keys_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } for foreign_key in foreign_keys @@ -229,8 +227,7 @@ async def populate_schema_tables(internal_db, db, schema_version): ).fetchall() indexes_to_insert.extend( { - "database_name": database_name, - "table_name": table_name, + **{"database_name": database_name, "table_name": table_name}, **dict(index), } for index in indexes @@ -262,7 +259,7 @@ async def populate_schema_tables(internal_db, db, schema_version): "catalog_tables", ): conn.execute( - f"DELETE FROM {table} WHERE database_name = ?", + "DELETE FROM {} WHERE database_name = ?".format(table), [database_name], ) conn.execute( diff --git a/datasette/utils/multipart.py b/datasette/utils/multipart.py index 13289b1c..cfa77486 100644 --- a/datasette/utils/multipart.py +++ b/datasette/utils/multipart.py @@ -11,10 +11,15 @@ Supports: import asyncio import shutil import tempfile -from collections.abc import Callable from dataclasses import dataclass, field from typing import ( Any, + Callable, + Dict, + List, + Optional, + Tuple, + Union, ) from urllib.parse import parse_qsl @@ -24,7 +29,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB DEFAULT_MAX_FIELDS = 1000 DEFAULT_MAX_FILES = 100 # If max_parts is not specified, it defaults to max_fields + max_files -DEFAULT_MAX_PARTS: int | None = None +DEFAULT_MAX_PARTS: Optional[int] = None DEFAULT_MAX_FIELD_SIZE = 100 * 1024 # 100KB DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024 # 1MB DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024 # 16KB @@ -35,6 +40,8 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024 # 50MB class MultipartParseError(Exception): """Raised when multipart parsing fails.""" + pass + @dataclass class UploadedFile: @@ -50,7 +57,7 @@ class UploadedFile: name: str filename: str - content_type: str | None + content_type: Optional[str] size: int _file: tempfile.SpooledTemporaryFile = field(repr=False) @@ -79,8 +86,7 @@ class UploadedFile: def __del__(self): try: self._file.close() - except Exception: # noqa: BLE001, S110 - # __del__ must never raise + except Exception: pass @@ -92,27 +98,27 @@ class FormData: """ def __init__(self): - self._data: list[tuple[str, str | UploadedFile]] = [] + self._data: List[Tuple[str, Union[str, UploadedFile]]] = [] - def append(self, key: str, value: str | UploadedFile) -> None: + def append(self, key: str, value: Union[str, UploadedFile]) -> None: """Add a key-value pair.""" self._data.append((key, value)) - def __getitem__(self, key: str) -> str | UploadedFile: + def __getitem__(self, key: str) -> Union[str, UploadedFile]: """Get the first value for a key.""" for k, v in self._data: if k == key: return v raise KeyError(key) - def get(self, key: str, default: Any = None) -> str | UploadedFile | None: + def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]: """Get the first value for a key, or default if not found.""" try: return self[key] except KeyError: return default - def getlist(self, key: str) -> list[str | UploadedFile]: + def getlist(self, key: str) -> List[Union[str, UploadedFile]]: """Get all values for a key.""" return [v for k, v in self._data if k == key] @@ -136,15 +142,15 @@ class FormData: """Return unique keys.""" return list(self) - def items(self) -> list[tuple[str, str | UploadedFile]]: + def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]: """Return all key-value pairs.""" return list(self._data) - def values(self) -> list[str | UploadedFile]: + def values(self) -> List[Union[str, UploadedFile]]: """Return all values.""" return [v for _, v in self._data] - def _uploaded_files(self) -> list[UploadedFile]: + def _uploaded_files(self) -> List[UploadedFile]: """Return UploadedFile instances contained in this form.""" return [v for _, v in self._data if isinstance(v, UploadedFile)] @@ -157,7 +163,7 @@ class FormData: for uploaded in self._uploaded_files(): try: uploaded.close_sync() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -166,7 +172,7 @@ class FormData: for uploaded in self._uploaded_files(): try: await uploaded.close() - except Exception: # noqa: BLE001, S110 + except Exception: # Best-effort cleanup; ignore close errors pass @@ -183,13 +189,13 @@ class FormData: await self.aclose() -def parse_content_disposition(header: str) -> dict[str, str | None]: +def parse_content_disposition(header: str) -> Dict[str, Optional[str]]: """ Parse Content-Disposition header value. Returns dict with 'name', 'filename' keys (filename may be None). """ - result: dict[str, str | None] = {"name": None, "filename": None} + result: Dict[str, Optional[str]] = {"name": None, "filename": None} # Split on semicolons, handling quoted strings parts = [] @@ -232,8 +238,7 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: from urllib.parse import unquote result["filename"] = unquote(encoded, encoding="utf-8") - except Exception: # noqa: BLE001, S110 - # Malformed RFC 5987 filename* - fall back to the plain filename + except Exception: pass continue @@ -245,19 +250,20 @@ def parse_content_disposition(header: str) -> dict[str, str | None]: if key == "name": result["name"] = value - # Only set filename if filename* hasn't already set it - elif key == "filename" and result["filename"] is None: - # Strip path components (security) - # Handle both Unix and Windows paths - value = value.replace("\\", "/") - if "/" in value: - value = value.rsplit("/", 1)[-1] - result["filename"] = value + elif key == "filename": + # Only set if filename* hasn't already set it + if result["filename"] is None: + # Strip path components (security) + # Handle both Unix and Windows paths + value = value.replace("\\", "/") + if "/" in value: + value = value.rsplit("/", 1)[-1] + result["filename"] = value return result -def parse_content_type(header: str) -> tuple[str, dict[str, str]]: +def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]: """ Parse Content-Type header value. @@ -301,7 +307,7 @@ class MultipartParser: max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, @@ -342,12 +348,12 @@ class MultipartParser: self._tempdir = tempfile.gettempdir() # Current part state - self.current_headers: dict[str, str] = {} - self.current_file: tempfile.SpooledTemporaryFile | None = None + self.current_headers: Dict[str, str] = {} + self.current_file: Optional[tempfile.SpooledTemporaryFile] = None self.current_body = bytearray() - self.current_name: str | None = None - self.current_filename: str | None = None - self.current_content_type: str | None = None + self.current_name: Optional[str] = None + self.current_filename: Optional[str] = None + self.current_content_type: Optional[str] = None def feed(self, chunk: bytes) -> None: """Feed a chunk of data to the parser.""" @@ -448,7 +454,7 @@ class MultipartParser: # Parse header try: line_str = line.decode("utf-8", errors="replace") - except UnicodeDecodeError: + except Exception: line_str = line.decode("latin-1") if ":" in line_str: @@ -475,9 +481,7 @@ class MultipartParser: if self.file_count > self.max_files: raise MultipartParseError("Too many files") if self.handle_files: - # Outlives this method - it is filled in across parser callbacks - # and then handed to the UploadedFile the caller consumes - self.current_file = tempfile.SpooledTemporaryFile( # noqa: SIM115 + self.current_file = tempfile.SpooledTemporaryFile( max_size=self.max_memory_file_size ) else: @@ -640,7 +644,7 @@ async def parse_form_data( max_request_size: int = DEFAULT_MAX_REQUEST_SIZE, max_fields: int = DEFAULT_MAX_FIELDS, max_files: int = DEFAULT_MAX_FILES, - max_parts: int | None = DEFAULT_MAX_PARTS, + max_parts: Optional[int] = DEFAULT_MAX_PARTS, max_field_size: int = DEFAULT_MAX_FIELD_SIZE, max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE, max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES, diff --git a/datasette/utils/permissions.py b/datasette/utils/permissions.py index 5a8ee8e2..fd1e41a1 100644 --- a/datasette/utils/permissions.py +++ b/datasette/utils/permissions.py @@ -2,9 +2,8 @@ from __future__ import annotations import json +from typing import Any, Dict, Iterable, List, Sequence, Tuple import sqlite3 -from collections.abc import Iterable, Sequence -from typing import Any from datasette.permissions import PermissionSQL from datasette.plugins import pm @@ -16,7 +15,7 @@ SKIP_PERMISSION_CHECKS = object() async def gather_permission_sql_from_hooks( *, datasette, actor: dict | None, action: str -) -> list[PermissionSQL] | object: +) -> List[PermissionSQL] | object: """Collect PermissionSQL objects from the permission_resources_sql hook. Ensures that each returned PermissionSQL has a populated ``source``. @@ -35,7 +34,7 @@ async def gather_permission_sql_from_hooks( hookimpls = hook_caller.get_hookimpls() hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action)) - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] actor_json = json.dumps(actor) if actor is not None else None actor_id = actor.get("id") if isinstance(actor, dict) else None @@ -72,7 +71,7 @@ def _iter_permission_sql_from_result( if isinstance(result, PermissionSQL): return [result] if isinstance(result, (list, tuple)): - collected: list[PermissionSQL] = [] + collected: List[PermissionSQL] = [] for item in result: collected.extend(_iter_permission_sql_from_result(item, action=action)) return collected @@ -91,7 +90,7 @@ def _iter_permission_sql_from_result( def build_rules_union( actor: dict | None, plugins: Sequence[PermissionSQL] -) -> tuple[str, dict[str, Any]]: +) -> Tuple[str, Dict[str, Any]]: """ Compose plugin SQL into a UNION ALL. @@ -103,10 +102,10 @@ def build_rules_union( The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent Plugin parameters should be prefixed with a unique identifier (e.g., source name). """ - parts: list[str] = [] + parts: List[str] = [] actor_json = json.dumps(actor) if actor else None actor_id = actor.get("id") if actor else None - params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} + params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id} for p in plugins: # No namespacing - just use plugin params as-is @@ -142,10 +141,10 @@ async def resolve_permissions_from_catalog( plugins: Sequence[Any], action: str, candidate_sql: str, - candidate_params: dict[str, Any] | None = None, + candidate_params: Dict[str, Any] | None = None, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions by embedding the provided *candidate_sql* in a CTE. @@ -169,8 +168,8 @@ async def resolve_permissions_from_catalog( - parent, child, allow, reason, source_plugin, depth - resource (rendered "/parent/child" or "/parent" or "/") """ - resolved_plugins: list[PermissionSQL] = [] - restriction_sqls: list[str] = [] + resolved_plugins: List[PermissionSQL] = [] + restriction_sqls: List[str] = [] for plugin in plugins: if callable(plugin) and not isinstance(plugin, PermissionSQL): @@ -399,11 +398,11 @@ async def resolve_permissions_with_candidates( db, actor: dict | None, plugins: Sequence[Any], - candidates: list[tuple[str, str | None]], + candidates: List[Tuple[str, str | None]], action: str, *, implicit_deny: bool = True, -) -> list[dict[str, Any]]: +) -> List[Dict[str, Any]]: """ Resolve permissions without any external candidate table by embedding the candidates as a UNION of parameterized SELECTs in a CTE. @@ -412,8 +411,8 @@ async def resolve_permissions_with_candidates( actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action """ # Build a small CTE for candidates. - cand_rows_sql: list[str] = [] - cand_params: dict[str, Any] = {} + cand_rows_sql: List[str] = [] + cand_params: Dict[str, Any] = {} for i, (parent, child) in enumerate(candidates): pkey = f"cand_p_{i}" ckey = f"cand_c_{i}" diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index d323f5d6..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE """ import os -from shutil import Error, copy, copy2, copystat +from shutil import copy, copy2, copystat, Error def _copytree( diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 334545bd..1be28982 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -413,12 +413,12 @@ def analyze_sql_tables( database=None, table=None, sqlite_schema=sqlite_schema, - target=f"{arg1} {arg2}" if arg2 is not None else arg1, + target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, source=source, ) return sqlite3.SQLITE_OK - action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}") + action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) record( "unknown", "unknown", @@ -521,7 +521,9 @@ def analyze_sql_tables( and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS ): return True - return bool(key_is_drop_table_delete(key)) + if key_is_drop_table_delete(key): + return True + return False def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d3926f6f..4743ae4c 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -100,7 +100,7 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str] schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return [] @@ -127,7 +127,7 @@ def _sqlite_table_type_from_schema( schema_table = _sqlite_schema_table(schema) try: row = conn.execute( - f"select type, sql from {schema_table} where name = ?", + "select type, sql from {} where name = ?".format(schema_table), (table,), ).fetchone() except sqlite3.DatabaseError: @@ -155,7 +155,7 @@ def _is_known_shadow_table( schema_table = _sqlite_schema_table(schema) try: rows = conn.execute( - f"select name, sql from {schema_table} where type = 'table'" + "select name, sql from {} where type = 'table'".format(schema_table) ).fetchall() except sqlite3.DatabaseError: return False @@ -174,7 +174,7 @@ def _sqlite_schema_table(schema: str | None) -> str: return "sqlite_master" if schema == "temp": return "sqlite_temp_master" - return f"{_quote_identifier(schema)}.sqlite_master" + return "{}.sqlite_master".format(_quote_identifier(schema)) def _quote_identifier(value: str) -> str: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index a8be47bf..de7e94af 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,7 +1,6 @@ -import json -from urllib.parse import urlencode - from asgiref.sync import async_to_sync +from urllib.parse import urlencode +import json # These wrapper classes pre-date the introduction of # datasette.client and httpx to Datasette. They could diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index bac3b39e..ed7e175f 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -1,7 +1,7 @@ +from dataclasses import dataclass import dataclasses import types import typing -from dataclasses import dataclass @dataclass(frozen=True) @@ -74,14 +74,16 @@ class Context: extra_class = table_extra_registry.classes_by_name[name] except KeyError: raise KeyError( - f"{cls.__name__}.{name} is declared with from_extra() but there is no " - "registered extra of that name" + "{}.{} is declared with from_extra() but there is no " + "registered extra of that name".format(cls.__name__, name) ) if cls.extras_scope is not None and not extra_class.available_for( cls.extras_scope ): raise ValueError( - f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is " - f"not available for scope {cls.extras_scope}" + "{}.{} is declared with from_extra() but the {} extra is " + "not available for scope {}".format( + cls.__name__, name, name, cls.extras_scope + ) ) return extra_class.description or "" diff --git a/datasette/views/base.py b/datasette/views/base.py index 48108cec..66e14a6d 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,20 +2,20 @@ import csv import hashlib import sys +from datasette.utils.asgi import Request from datasette.utils import ( + add_cors_headers, EscapeHtmlWriter, InvalidSql, LimitedWriter, - add_cors_headers, path_from_row_pks, path_with_format, sqlite3, ) from datasette.utils.asgi import ( AsgiStream, - BadRequest, - Request, Response, + BadRequest, ) @@ -129,10 +129,12 @@ class BaseView: template = environment.select_template(templates) template_context = { **context, - "select_templates": [ - f"{'*' if template_name == template.name else ''}{template_name}" - for template_name in templates - ], + **{ + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + }, } headers = {} if self.has_json_alternate: @@ -149,7 +151,9 @@ class BaseView: template_context["alternate_url_json"] = alternate_url_json headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) return Response.html( @@ -180,7 +184,9 @@ async def stream_csv(datasette, fetch_data, request, database): stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ - f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key) + "{}=1".format(key) + for key in ("_nofacet", "_nocount") + if not request.args.get(key) ] if extra_parameters: # Replace request object with a new one with modified scope @@ -210,6 +216,9 @@ async def stream_csv(datasette, fetch_data, request, database): except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + except DatasetteError: raise @@ -316,9 +325,8 @@ async def stream_csv(datasette, fetch_data, request, database): else: new_row.append(cell) await writer.writerow(new_row) - except Exception as ex: # noqa: BLE001 - # Streaming CSV: report the error into the response body and stop - sys.stderr.write(f"Caught this error: {ex}\n") + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() await r.write(str(ex)) return diff --git a/datasette/views/database.py b/datasette/views/database.py index f54ffd38..11646f45 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,52 +1,49 @@ +from dataclasses import asdict, dataclass, field +from urllib.parse import parse_qsl, urlencode import asyncio import hashlib import itertools import json +import markupsafe import os import textwrap -from dataclasses import asdict, dataclass, field -from urllib.parse import parse_qsl, urlencode - -import markupsafe +from datasette.extras import extra_names_from_request, ExtraScope from datasette.database import QueryInterrupted -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, stored_query_to_dict +from datasette.write_sql import QueryWriteRejected from datasette.utils import ( - InvalidSql, add_cors_headers, await_me_maybe, - call_with_supported_arguments, error_body, + call_with_supported_arguments, + named_parameters as derive_named_parameters, format_bytes, - is_url, make_slot_function, + tilde_decode, + to_css_class, + validate_sql_select, + is_url, path_with_added_args, path_with_format, path_with_removed_args, sqlite3, - tilde_decode, - to_css_class, truncate_url, - validate_sql_select, + InvalidSql, ) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response -from datasette.write_sql import QueryWriteRejected +from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden +from datasette.plugins import pm -from . import Context from .base import DatasetteError, View, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns -from .table_create_alter import _create_table_ui_context from .table_extras import ( QueryExtraContext, resolve_query_extras, table_extra_registry, ) +from .table_create_alter import _create_table_ui_context +from . import Context @dataclass @@ -103,7 +100,7 @@ class DatabaseView(View): return response if format_ not in ("html", "json"): - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) metadata = await datasette.get_database_metadata(database) @@ -167,7 +164,7 @@ class DatabaseView(View): "label": "Create table", "description": "Create a new table in this database.", "attrs": { - "aria-label": f"Create table in {database}", + "aria-label": "Create table in {}".format(database), "data-database-action": "create-table", }, } @@ -274,7 +271,9 @@ class DatabaseView(View): view_name="database", ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) @@ -557,7 +556,7 @@ async def database_download(request, datasette): if datasette.cors: add_cors_headers(headers) if db.hash: - etag = f'"{db.hash}"' + etag = '"{}"'.format(db.hash) headers["Etag"] = etag # Has user seen this already? if_none_match = request.headers.get("if-none-match") @@ -665,9 +664,8 @@ class QueryView(View): ).first() if message_result: message = message_result[0] - except Exception as ex: # noqa: BLE001 - # Stored-query on_success_message_sql is user-authored - message = f"Error running on_success_message_sql: {ex}" + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: if stored_query.on_success_message: @@ -681,8 +679,7 @@ class QueryView(View): redirect_url = stored_query.on_success_redirect ok = True - except Exception as ex: # noqa: BLE001 - # Stored-query execution is user-authored SQL + except Exception as ex: message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR redirect_url = stored_query.on_error_redirect @@ -816,16 +813,16 @@ class QueryView(View): rows = results.rows except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -841,6 +838,8 @@ class QueryView(View): columns = [] except (sqlite3.OperationalError, InvalidSql) as ex: raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) except DatasetteError: raise @@ -862,7 +861,7 @@ class QueryView(View): return data, None, None return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers: + elif format_ in datasette.renderers.keys(): if not sql: raise DatasetteError("?sql= is required", status=400) data = {"ok": True, "rows": rows, "columns": columns} @@ -954,7 +953,9 @@ class QueryView(View): } headers.update( { - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) } ) metadata = await query_metadata() @@ -1035,7 +1036,9 @@ class QueryView(View): + "?" + urlencode( { - "sql": sql, + **{ + "sql": sql, + }, **named_parameter_values, } ) @@ -1137,7 +1140,7 @@ class QueryView(View): headers=headers, ) else: - assert False, f"Invalid format: {format_}" + assert False, "Invalid format: {}".format(format_) if datasette.cors: add_cors_headers(r.headers) return r @@ -1238,7 +1241,7 @@ async def display_rows(datasette, database, request, rows, columns): '<Binary: {:,} byte{}>'.format( blob_url, ( - f' title="{formatted}"' + ' title="{}"'.format(formatted) if "bytes" not in formatted else "" ), diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 3dfb810c..dd35b127 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -8,8 +8,8 @@ from datasette.utils.asgi import Response from .base import BaseView from .database import display_rows as display_query_rows from .query_helpers import ( - SQL_PARAMETER_FORM_PREFIX, QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -31,7 +31,15 @@ WRITE_TEMPLATE_LABELS = { "delete": "Delete rows", } WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) -CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _parameter_names(columns): @@ -41,11 +49,11 @@ def _parameter_names(columns): base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) base = base.strip("_") or "value" if base[0].isdigit(): - base = f"p_{base}" + base = "p_{}".format(base) name = base index = 2 while name in seen: - name = f"{base}_{index}" + name = "{}_{}".format(base, index) index += 1 seen.add(name) names[column] = name @@ -57,7 +65,7 @@ def _quote_identifier(identifier): def _preferred_where_column(table, columns): - lower_table_id = f"{table.lower()}_id" + lower_table_id = "{}_id".format(table.lower()) return ( next((column for column in columns if column.lower() == "id"), None) or next( @@ -82,15 +90,17 @@ def _insert_template_sql(table, columns): auto_pk = _auto_incrementing_primary_key(columns) insert_columns = [column for column in column_names if column != auto_pk] if not insert_columns: - return f"insert into {_quote_identifier(table)}\ndefault values" + return "insert into {}\ndefault values".format(_quote_identifier(table)) names = _parameter_names(insert_columns) return "\n".join( ( - f"insert into {_quote_identifier(table)} (", - ",\n".join(f" {_quote_identifier(column)}" for column in insert_columns), + "insert into {} (".format(_quote_identifier(table)), + ",\n".join( + " {}".format(_quote_identifier(column)) for column in insert_columns + ), ")", "values (", - ",\n".join(f" :{names[column]}" for column in insert_columns), + ",\n".join(" :{}".format(names[column]) for column in insert_columns), ")", ) ) @@ -104,14 +114,18 @@ def _update_template_sql(table, columns): if not set_columns: return "\n".join( ( - f"update {_quote_identifier(table)}", - f"set {_quote_identifier(where_column)} = :new_{names[where_column]}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "update {}".format(_quote_identifier(table)), + "set {} = :new_{}".format( + _quote_identifier(where_column), names[where_column] + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) return "\n".join( ( - f"update {_quote_identifier(table)}", + "update {}".format(_quote_identifier(table)), "set " + ",\n".join( "{}{} = :{}".format( @@ -121,7 +135,9 @@ def _update_template_sql(table, columns): ) for index, column in enumerate(set_columns) ), - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) @@ -132,8 +148,10 @@ def _delete_template_sql(table, columns): where_column = _preferred_where_column(table, column_names) return "\n".join( ( - f"delete from {_quote_identifier(table)}", - f"where {_quote_identifier(where_column)} = :{names[where_column]}", + "delete from {}".format(_quote_identifier(table)), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), ) ) diff --git a/datasette/views/index.py b/datasette/views/index.py index f73ee38a..67296cd1 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,11 +2,11 @@ import json from datasette.plugins import pm from datasette.utils import ( - UNSTABLE_API_MESSAGE, - CustomJSONEncoder, add_cors_headers, await_me_maybe, make_slot_function, + CustomJSONEncoder, + UNSTABLE_API_MESSAGE, ) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -46,15 +46,15 @@ class IndexView(BaseView): databases = [] # Iterate over allowed databases instead of all databases - for name, allowed_db in allowed_db_dict.items(): + for name in allowed_db_dict.keys(): db = self.ds.databases[name] - database_private = allowed_db.private + database_private = allowed_db_dict[name].private # Get allowed tables/views for this database allowed_for_db = tables_by_db.get(name, {}) # Get table names from allowed set instead of db.table_names() - table_names = [child_name for child_name in allowed_for_db] + table_names = [child_name for child_name in allowed_for_db.keys()] hidden_table_names = set(await db.hidden_table_names()) @@ -99,7 +99,7 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - if table in tables: + if table in tables.keys(): count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) tables[table]["num_relationships_for_sorting"] = count @@ -121,7 +121,8 @@ class IndexView(BaseView): # Only add views if this is less than TRUNCATE_AT if len(tables_and_views_truncated) < TRUNCATE_AT: num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated) - tables_and_views_truncated.extend(views[:num_views_to_add]) + for view in views[:num_views_to_add]: + tables_and_views_truncated.append(view) databases.append( { diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 725d9cdb..588891d4 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,19 +5,6 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, ) -from datasette.utils import ( - InvalidSql, - escape_sqlite, - parse_size_limit, - path_from_row_pks, - sqlite3, - validate_sql_select, -) -from datasette.utils import ( - named_parameters as derive_named_parameters, -) -from datasette.utils.asgi import Forbidden -from datasette.utils.sql_analysis import Operation, SQLAnalysis from datasette.write_sql import ( IgnoreWriteSqlOperation, QueryWriteRejected, @@ -25,6 +12,17 @@ from datasette.write_sql import ( decision_for_write_sql_operation, operation_is_write, ) +from datasette.utils import ( + parse_size_limit, + named_parameters as derive_named_parameters, + escape_sqlite, + path_from_row_pks, + sqlite3, + validate_sql_select, + InvalidSql, +) +from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -93,7 +91,7 @@ def _as_optional_bool(value, name): return True if lowered in {"0", "false", "f", "no", "off"}: return False - raise QueryValidationError(f"{name} must be 0 or 1") + raise QueryValidationError("{} must be 0 or 1".format(name)) def _query_list_limit(value, default, maximum): @@ -173,7 +171,7 @@ async def _json_or_form_payload(request): try: return json.loads(body or b"{}"), True except json.JSONDecodeError as e: - raise QueryValidationError(f"Invalid JSON: {e}") + raise QueryValidationError("Invalid JSON: {}".format(e)) return await request.post_vars(), False @@ -194,7 +192,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex is_write = _analysis_is_write(analysis) if is_write: @@ -295,7 +293,8 @@ def _coerce_execute_write_payload(data, is_json): for key, value in data.items(): if key in {"sql", "csrftoken", "_json"}: continue - key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX) + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") @@ -315,7 +314,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): try: analysis = await db.analyze_sql(sql, params) except sqlite3.DatabaseError as ex: - raise QueryValidationError(f"Could not analyze query: {ex}") from ex + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex if not _analysis_is_write(analysis): raise QueryValidationError( "Use /-/query for read-only SQL; this endpoint only executes writes" @@ -497,7 +496,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): ) try: result = await db.execute( - f"select {select} from {escape_sqlite(table)} where rowid = ?", + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), [lastrowid], ) except sqlite3.DatabaseError: diff --git a/datasette/views/row.py b/datasette/views/row.py index b1388299..c90a3bbe 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,35 +8,34 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils +from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response from datasette.database import QueryInterrupted -from datasette.events import DeleteRowEvent, UpdateRowEvent -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.plugins import pm +from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource +from .base import BaseView, DatasetteError, stream_csv from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, + CustomRow, decode_write_json_row, - escape_sqlite, + InvalidSql, make_slot_function, path_from_row_pks, path_with_format, path_with_removed_args, - sqlite3, to_css_class, + escape_sqlite, + sqlite3, + WriteJsonValueError, ) -from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response - +from datasette.plugins import pm +from datasette.extras import extra_names_from_request, ExtraScope from . import Context, from_extra -from .base import BaseView, DatasetteError, stream_csv from .table import ( - _table_page_data, display_columns_and_rows, + _table_page_data, row_label_from_label_column, ) from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -188,16 +187,16 @@ class RowView(BaseView): data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted as ex: raise DatasetteError( - textwrap.dedent(f""" + textwrap.dedent("""

SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

- + - """).strip(), + """.format(markupsafe.escape(ex.sql))).strip(), title="SQL Interrupted", status=400, message_is_html=True, @@ -208,13 +207,15 @@ class RowView(BaseView): ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) except DatasetteError: raise end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - if format_ in self.ds.renderers: + if format_ in self.ds.renderers.keys(): # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -257,7 +258,7 @@ class RowView(BaseView): if status_code is not None: response.status = status_code else: - raise NotFound(f"Invalid format: {format_}") + raise NotFound("Invalid format: {}".format(format_)) ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): @@ -372,7 +373,9 @@ class RowView(BaseView): view_name=self.name, ), headers={ - "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"' + "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) }, ) @@ -497,7 +500,7 @@ class RowView(BaseView): row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) row_action_permissions = {} if is_table and db.is_mutable: @@ -510,7 +513,7 @@ class RowView(BaseView): row_actions = [] if row_action_permissions.get("update-row"): attrs = { - "aria-label": f"Edit row {row_action_label}", + "aria-label": "Edit row {}".format(row_action_label), "data-row": row_path, "data-row-action": "edit", } @@ -526,7 +529,7 @@ class RowView(BaseView): ) if row_action_permissions.get("delete-row"): attrs = { - "aria-label": f"Delete row {row_action_label}", + "aria-label": "Delete row {}".format(row_action_label), "data-row": row_path, "data-row-action": "delete", } @@ -676,7 +679,7 @@ class RowView(BaseView): key, ",".join(pk_values), ) - foreign_key_tables.append({**fk, "count": count, "link": link}) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) return foreign_key_tables @@ -702,21 +705,23 @@ async def _row_flash_message(db, action, resolved, row=None): if label: label = _truncated_row_flash_label(label) if label and label != pk_label: - return f"{action} row {pk_label} ({label})" - return f"{action} row {pk_label}" + return "{} row {} ({})".format(action, pk_label, label) + return "{} row {}".format(action, pk_label) async def _resolve_row_and_check_permission(datasette, request, permission): - from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound + from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound try: resolved = await datasette.resolve_row(request) except DatabaseNotFound as e: - return False, Response.error([f"Database not found: {e.database_name}"], 404) + return False, Response.error( + ["Database not found: {}".format(e.database_name)], 404 + ) except TableNotFound as e: - return False, Response.error([f"Table not found: {e.table}"], 404) + return False, Response.error(["Table not found: {}".format(e.table)], 404) except RowNotFound as e: - return False, Response.error([f"Record not found: {e.pk_values}"], 404) + return False, Response.error(["Record not found: {}".format(e.pk_values)], 404) # Ensure user has permission to delete this row if not await datasette.allowed( @@ -748,8 +753,7 @@ class RowDeleteView(BaseView): try: await resolved.db.execute_write_fn(delete_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s + except Exception as e: return Response.error([str(e)], 400) await self.ds.track_event( @@ -789,7 +793,7 @@ class RowUpdateView(BaseView): try: data = await request.json() except json.JSONDecodeError as e: - return Response.error([f"Invalid JSON: {e}"]) + return Response.error(["Invalid JSON: {}".format(e)]) except PayloadTooLarge as e: return Response.error([str(e)], 413) @@ -832,8 +836,7 @@ class RowUpdateView(BaseView): try: await resolved.db.execute_write_fn(update_row, request=request) - except Exception as e: # noqa: BLE001 - # TODO: narrow to expected write errors so Datasette bugs surface as 500s + except Exception as e: return Response.error([str(e)], 400) result = {"ok": True} diff --git a/datasette/views/special.py b/datasette/views/special.py index a77f221f..28d34208 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,25 +1,23 @@ import json import logging -import secrets -import urllib - -from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent from datasette.jump import JumpSQL, namespace_sql_params from datasette.plugins import pm +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource +from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( UNSTABLE_API_MESSAGE, actor_matches_allow, + parse_size_limit, add_cors_headers, await_me_maybe, error_body, - parse_size_limit, - tilde_decode, tilde_encode, + tilde_decode, ) -from datasette.utils.asgi import Forbidden, Response - from .base import BaseView, View +import secrets +import urllib logger = logging.getLogger(__name__) @@ -181,7 +179,9 @@ class AutocompleteDebugView(BaseView): ) context.update( { - "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete", + "autocomplete_url": "{}/-/autocomplete".format( + self.ds.urls.table(database_name, table_name) + ), "label_column": await db.label_column_for_table(table_name), } ) @@ -420,11 +420,8 @@ class AllowedResourcesView(BaseView): row["reason"] = resource.reasons allowed_rows.append(row) - except Exception: # noqa: BLE001 - # Returns empty results if the catalog tables don't exist yet, but - # also swallows the AttributeError raised for instance-level actions - # such as view-instance, which have no resource_class. - # TODO: handle that case explicitly and narrow this to sqlite3.Error + except Exception: + # If catalog tables don't exist yet, return empty results return ( { "ok": True, @@ -526,7 +523,7 @@ class PermissionRulesView(BaseView): from datasette.utils.actions_sql import build_permission_rules_sql - union_sql, union_params, _restriction_sqls = await build_permission_rules_sql( + union_sql, union_params, restriction_sqls = await build_permission_rules_sql( self.ds, actor, action ) await self.ds.refresh_schemas() @@ -939,7 +936,7 @@ class ApiExplorerView(BaseView): tables.append({"name": table, "links": table_links}) table_links.append( { - "label": f"Get rows for {table}", + "label": "Get rows for {}".format(table), "method": "GET", "path": self.ds.urls.table(name, table, format="json"), } @@ -959,7 +956,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/insert", "method": "POST", - "label": f"Insert rows into {table}", + "label": "Insert rows into {}".format(table), "json": { "rows": [ { @@ -973,7 +970,7 @@ class ApiExplorerView(BaseView): { "path": self.ds.urls.table(name, table) + "/-/upsert", "method": "POST", - "label": f"Upsert rows into {table}", + "label": "Upsert rows into {}".format(table), "json": { "rows": [ { @@ -1003,7 +1000,7 @@ class ApiExplorerView(BaseView): table_links.append( { "path": self.ds.urls.table(name, table) + "/-/drop", - "label": f"Drop table {table}", + "label": "Drop table {}".format(table), "json": {"confirm": False}, "method": "POST", } @@ -1020,7 +1017,7 @@ class ApiExplorerView(BaseView): database_links.append( { "path": self.ds.urls.database(name) + "/-/create", - "label": f"Create table in {name}", + "label": "Create table in {}".format(name), "json": { "table": "new_table", "columns": [ diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 0bbe9f38..d64f37d2 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -124,7 +124,7 @@ class QueryListView(BaseView): pairs.append(("_next", page.next)) next_url = self.ds.absolute_url( request, - f"{request.path}?{urlencode(pairs)}", + "{}?{}".format(request.path, urlencode(pairs)), ) current_filters = { @@ -415,7 +415,7 @@ class QueryDefinitionView(BaseView): query_name = tilde_decode(request.url_vars["query"]) query = await self.ds.get_query(db.name, query_name) if query is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="view-query", resource=QueryResource(db.name, query_name), @@ -439,7 +439,7 @@ class QueryUpdateView(BaseView): query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -532,7 +532,7 @@ class QueryEditView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="update-query", resource=QueryResource(db.name, query_name), @@ -545,7 +545,7 @@ class QueryEditView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="update-query", resource=QueryResource(db.name, query_name), @@ -629,7 +629,7 @@ class QueryDeleteView(BaseView): async def get(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) await self.ds.ensure_permission( action="delete-query", resource=QueryResource(db.name, query_name), @@ -653,7 +653,7 @@ class QueryDeleteView(BaseView): async def post(self, request): db, query_name, existing = await self._load(request) if existing is None: - return Response.error([f"Query not found: {query_name}"], 404) + return Response.error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( action="delete-query", resource=QueryResource(db.name, query_name), @@ -665,13 +665,13 @@ class QueryDeleteView(BaseView): ["Trusted queries cannot be deleted using the API"], 403 ) - _data, is_json = await _json_or_form_payload(request) + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) if is_json: return Response.json({"ok": True}) self.ds.add_message( request, - f"Query “{existing.title or query_name}” deleted", + "Query “{}” deleted".format(existing.title or query_name), self.ds.INFO, ) return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/datasette/views/table.py b/datasette/views/table.py index 3eb80854..f7edd744 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,51 +3,48 @@ import itertools import json import urllib import urllib.parse -from dataclasses import dataclass, field import markupsafe -import sqlite_utils -from datasette import tracer from datasette.column_types import SQLiteType -from datasette.database import QueryInterrupted +from datasette.extras import extra_names_from_request +from datasette.plugins import pm from datasette.events import ( AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, ) -from datasette.extras import ExtraScope, extra_names_from_request -from datasette.filters import Filters -from datasette.plugins import pm +from datasette.database import QueryInterrupted +from datasette import tracer from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( - CustomJSONEncoder, - CustomRow, - InvalidSql, - WriteJsonValueError, add_cors_headers, - append_querystring, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, + CustomRow, + append_querystring, compound_keys_after_sql, decode_write_json_rows, + format_bytes, + make_slot_function, + tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, - make_slot_function, path_from_row_pks, path_with_added_args, path_with_format, path_with_removed_args, path_with_replaced_args, - sqlite3, - tilde_encode, to_css_class, truncate_url, urlsafe_components, value_as_boolean, + InvalidSql, + WriteJsonValueError, + sqlite3, ) from datasette.utils.asgi import ( BadRequest, @@ -57,7 +54,11 @@ from datasette.utils.asgi import ( Request, Response, ) +from datasette.filters import Filters +import sqlite_utils +from dataclasses import dataclass, field +from datasette.extras import ExtraScope from . import Context, from_extra from .base import BaseView, DatasetteError, stream_csv from .database import QueryView @@ -535,7 +536,7 @@ async def _table_insert_ui( columns.append(column_data) data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/insert", + "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, "bulkColumns": bulk_columns, @@ -543,8 +544,8 @@ async def _table_insert_ui( "maxInsertRows": datasette.setting("max_insert_rows"), } if can_update: - data["upsertPath"] = ( - f"{datasette.urls.table(database_name, table_name)}/-/upsert" + data["upsertPath"] = "{}/-/upsert".format( + datasette.urls.table(database_name, table_name) ) return data @@ -603,7 +604,7 @@ async def _table_alter_ui( columns.append(column_data) data = { - "path": f"{datasette.urls.table(database_name, table_name)}/-/alter", + "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)), "tableName": table_name, "columns": columns, "primaryKeys": pks, @@ -629,7 +630,9 @@ async def _table_alter_ui( actor=request.actor, ) if can_drop_table: - data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop" + data["dropPath"] = "{}/-/drop".format( + datasette.urls.table(database_name, table_name) + ) return data @@ -725,10 +728,12 @@ async def display_columns_and_rows( row_label = row_label_from_label_column(row, label_column) row_action_label = pk_path if row_label and row_label != pk_path: - row_action_label = f"{pk_path} {row_label}" + row_action_label = "{} {}".format(pk_path, row_label) table_path = datasette.urls.table(database_name, table_name) - row_link = ( - f'{markupsafe.escape(pk_path)!s}' + row_link = '{flat_pks}'.format( + table_path=table_path, + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=row_path, ) edit_icon = ( '

{error}

' in response2.text + assert '

{}

'.format(error) in response2.text else: # Check create-token event event = last_event(app_client.ds) @@ -231,7 +228,7 @@ def test_auth_create_token( # And test that token response3 = app_client.get( "/-/actor.json", - headers={"Authorization": "Bearer {}".format(f"dstok_{token}")}, + headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))}, ) assert response3.status == 200 assert response3.json["actor"]["id"] == "test" @@ -244,7 +241,7 @@ async def test_auth_create_token_not_allowed_for_tokens(ds_client): ) response = await ds_client.get( "/-/create-token", - headers={"Authorization": f"Bearer dstok_{ds_tok}"}, + headers={"Authorization": "Bearer dstok_{}".format(ds_tok)}, ) assert response.status_code == 403 @@ -289,12 +286,12 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work): elif scenario == "invalid_token": token = "invalid" if token: - token = f"dstok_{token}" + token = "dstok_{}".format(token) if scenario == "allow_signed_tokens_off": ds_client.ds._settings["allow_signed_tokens"] = False headers = {} if token: - headers["Authorization"] = f"Bearer {token}" + headers["Authorization"] = "Bearer {}".format(token) response = await ds_client.get("/-/actor.json", headers=headers) try: if should_work: @@ -341,7 +338,7 @@ def test_cli_create_token(app_client, expires): assert details.keys() == expected_keys assert details["a"] == "test" response = app_client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) if expires is None or expires > 0: expected_actor = { diff --git a/tests/test_base_view.py b/tests/test_base_view.py index b46f7ce1..c1b0cf20 100644 --- a/tests/test_base_view.py +++ b/tests/test_base_view.py @@ -1,10 +1,8 @@ -import json - -import pytest - +from datasette.views.base import View from datasette import Request, Response from datasette.app import Datasette -from datasette.views.base import View +import json +import pytest class GetView(View): diff --git a/tests/test_cli.py b/tests/test_cli.py index fbd4a8a9..cbd8edad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,28 +1,23 @@ +from .fixtures import ( + make_app_client, + TestClient as _TestClient, + EXPECTED_PLUGINS, +) +from datasette.app import SETTINGS +from datasette.plugins import DEFAULT_PLUGINS, pm +from datasette.cli import cli, serve +from datasette.version import __version__ +from datasette.utils import tilde_encode +from datasette.utils.sqlite import sqlite3 +from click.testing import CliRunner import io import json import pathlib +import pytest import sys import textwrap from unittest import mock -import pytest -from click.testing import CliRunner - -from datasette.app import SETTINGS -from datasette.cli import cli, serve -from datasette.plugins import DEFAULT_PLUGINS, pm -from datasette.utils import tilde_encode -from datasette.utils.sqlite import sqlite3 -from datasette.version import __version__ - -from .fixtures import ( - EXPECTED_PLUGINS, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) - def test_inspect_cli(app_client): runner = CliRunner() @@ -465,7 +460,7 @@ def test_serve_create(tmpdir): @pytest.mark.parametrize("argument", ("-c", "--config")) @pytest.mark.parametrize("format_", ("json", "yaml")) def test_serve_config(tmpdir, argument, format_): - config_path = tmpdir / f"datasette.{format_}" + config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( ( "settings:\n default_page_size: 5\n" @@ -518,13 +513,13 @@ def test_weird_database_names(tmpdir, filename): result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] - expected_link = ( - f'{filename_no_stem}' + expected_link = '{}'.format( + tilde_encode(filename_no_stem), filename_no_stem ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"] + cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] ) assert result2.exit_code == 0, result2.output diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 01b84f59..fe9416d6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,10 +1,8 @@ -import json -import textwrap - -from click.testing import CliRunner - from datasette.cli import cli from datasette.plugins import pm +from click.testing import CliRunner +import textwrap +import json def test_serve_with_get(tmp_path_factory): @@ -46,9 +44,9 @@ def test_serve_with_get(tmp_path_factory): # Annoyingly that new test plugin stays resident - we need # to manually unregister it to avoid conflict with other tests - to_unregister = next( + to_unregister = [ p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py" - ) + ][0] pm.unregister(to_unregister) diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index b7604bb8..47f23c08 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,7 +1,6 @@ -import socket - import httpx import pytest +import socket @pytest.mark.serial diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 50c6daed..cd308ec9 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,11 +1,7 @@ import json import logging -import time -import markupsafe -import pytest from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -13,7 +9,11 @@ from datasette.column_types import ( ) from datasette.hookspecs import hookimpl from datasette.plugins import pm -from datasette.utils import StartupError, error_body, sqlite3 +from datasette.utils import error_body, sqlite3 +from datasette.utils import StartupError +import markupsafe +import pytest +import time @pytest.fixture @@ -104,7 +104,7 @@ def write_token(ds, actor_id="root", permissions=None): def _headers(token): return { - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", } diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 00540464..636b17eb 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,12 +1,10 @@ import json import pathlib - import pytest from datasette.app import Datasette -from datasette.utils import StartupError from datasette.utils.sqlite import sqlite3 - +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient PLUGIN = """ diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index ffd0870c..11e53224 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -1,9 +1,7 @@ -import sqlite3 -import urllib - -from click.testing import CliRunner - from datasette.cli import cli +from click.testing import CliRunner +import urllib +import sqlite3 def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): @@ -42,7 +40,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): db_dir = tmp_path_factory.mktemp("dbs") dbs = [] for i in range(11): - path = str(db_dir / f"db_{i}.db") + path = str(db_dir / "db_{}.db".format(i)) conn = sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 6c78f69d..2fcfb216 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -44,7 +44,7 @@ async def _run_middleware(scope): await mw(scope, None, send) if inner_called: return ("allowed",) - start = next(m for m in sent if m["type"] == "http.response.start") + start = [m for m in sent if m["type"] == "http.response.start"][0] return ("blocked", start["status"]) diff --git a/tests/test_csv.py b/tests/test_csv.py index 7758a3c0..a2f03776 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,9 +1,7 @@ -import urllib.parse - -import pytest -from bs4 import BeautifulSoup as Soup - from datasette.app import Datasette +from bs4 import BeautifulSoup as Soup +import pytest +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 32cfc43d..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -1,7 +1,5 @@ import pathlib - import pytest - from .fixtures import make_app_client TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index f456a17f..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -1,5 +1,4 @@ import pytest - from datasette.app import Datasette from datasette.resources import DatabaseResource, TableResource diff --git a/tests/test_docs.py b/tests/test_docs.py index 16df6a46..0bcb5e62 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,22 +2,20 @@ Tests to ensure certain things are documented. """ -import re -from pathlib import Path - -import pytest - -import datasette.fixtures # noqa: F401 from datasette import app, utils +import datasette.fixtures # noqa: F401 from datasette.app import Datasette from datasette.filters import Filters +from pathlib import Path +import pytest +import re docs_path = Path(__file__).parent.parent / "docs" label_re = re.compile(r"\.\. _([^\s:]+):") def get_headings(content, underline="-"): - heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n") + heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline)) return {h[0] for h in heading_re.findall(content)} diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index 4a0014b4..613160ac 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -1,10 +1,9 @@ # fmt: off # -- start datasette_with_plugin_fixture -- -import pytest -import pytest_asyncio - from datasette import hookimpl from datasette.app import Datasette +import pytest +import pytest_asyncio @pytest_asyncio.fixture diff --git a/tests/test_error_shape.py b/tests/test_error_shape.py index 94c9a7c9..768814fd 100644 --- a/tests/test_error_shape.py +++ b/tests/test_error_shape.py @@ -17,10 +17,8 @@ present and the legacy "title" key must not be. https://github.com/simonw/datasette/issues - 1.0 API consistency """ -import time - import pytest - +import time from datasette.app import Datasette from datasette.utils import sqlite3 @@ -88,7 +86,7 @@ async def test_write_api_validation_error_shape(ds_error_shape): "/data/docs/-/insert", json={"rows": [{"nope": 1}, {"also_nope": 2}]}, headers={ - "Authorization": f"Bearer {token}", + "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", }, ) @@ -412,7 +410,7 @@ async def test_expired_token_returns_401(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) data = assert_canonical_error(response, 401) assert "expired" in data["error"].lower() @@ -448,7 +446,7 @@ async def test_valid_token_still_authenticates(ds_error_shape): ) ) response = await ds_error_shape.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) assert response.status_code == 200 assert response.json()["actor"]["id"] == "root" @@ -479,7 +477,7 @@ async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory): ds.sign({"a": "root", "t": int(time.time())}, namespace="token") ) response = await ds.client.get( - "/-/actor.json", headers={"Authorization": f"Bearer {token}"} + "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)} ) data = assert_canonical_error(response, 401) assert "not enabled" in data["error"] @@ -644,7 +642,7 @@ async def test_query_list_size_rejects_non_integer(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("endpoint", ("allowed", "rules")) async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint): - base = f"/-/{endpoint}.json?action=view-instance" + base = "/-/{}.json?action=view-instance".format(endpoint) ok = await ds_error_shape.client.get( base + "&_size=1&_page=1", actor={"id": "root"} ) diff --git a/tests/test_extras.py b/tests/test_extras.py index 4e008926..73b4965e 100644 --- a/tests/test_extras.py +++ b/tests/test_extras.py @@ -1,5 +1,4 @@ import asyncio -from typing import ClassVar import pytest @@ -8,7 +7,7 @@ from datasette.extras import Extra, ExtraRegistry, ExtraScope class SlowValueExtra(Extra): description = "Returns context['value'], optionally slowly" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context["slow"]: @@ -18,7 +17,7 @@ class SlowValueExtra(Extra): class DependentExtra(Extra): description = "Depends on slow_value" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} async def resolve(self, context, slow_value): return slow_value + 1 @@ -26,7 +25,7 @@ class DependentExtra(Extra): class InternalOnlyExtra(Extra): description = "Internal extra for HTML templates only" - scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE} + scopes = {ExtraScope.TABLE} public = False async def resolve(self, context): @@ -53,7 +52,7 @@ def _registered_extra_classes(): @pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key()) def test_registered_extras_have_descriptions(cls): # Every registered extra is part of the documented template/JSON contract - assert cls.description, f"{cls.__name__} is missing a description" + assert cls.description, "{} is missing a description".format(cls.__name__) def test_registry_is_built_once_per_scope(): diff --git a/tests/test_facets.py b/tests/test_facets.py index b8eb6e61..8c22ffce 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,14 +1,11 @@ -import json - -import pytest - from datasette.app import Datasette from datasette.database import Database -from datasette.facets import ArrayFacet, ColumnFacet, DateFacet, Facet -from datasette.utils import detect_json1 +from datasette.facets import Facet, ColumnFacet, ArrayFacet, DateFacet from datasette.utils.asgi import Request - +from datasette.utils import detect_json1 from .fixtures import make_app_client +import json +import pytest @pytest.mark.asyncio @@ -540,7 +537,7 @@ async def test_facet_size(): for j in range(1, 4): await db.execute_write( "insert into neighbourhoods (city, neighbourhood) values (?, ?)", - [f"City {i}", f"Neighbourhood {j}"], + ["City {}".format(i), "Neighbourhood {}".format(j)], ) response = await ds.client.get( "/test_facet_size/neighbourhoods.json?_extra=suggested_facets" diff --git a/tests/test_filters.py b/tests/test_filters.py index 8d0f3512..eda9e9a1 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,7 +1,6 @@ -import pytest - -from datasette.filters import Filters, search_filters, through_filters, where_filters +from datasette.filters import Filters, through_filters, where_filters, search_filters from datasette.utils.asgi import Request +import pytest @pytest.mark.parametrize( diff --git a/tests/test_html.py b/tests/test_html.py index 6a7b4907..b4c47d80 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1,19 +1,16 @@ +from bs4 import BeautifulSoup as Soup +from datasette.app import Datasette +from datasette.utils import allowed_pragmas +from .fixtures import make_app_client +from .utils import assert_footer_links, inner_html import copy import hashlib import json import pathlib +import pytest import re import urllib.parse -import pytest -from bs4 import BeautifulSoup as Soup - -from datasette.app import Datasette -from datasette.utils import allowed_pragmas - -from .fixtures import make_app_client -from .utils import assert_footer_links, inner_html - def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") @@ -145,7 +142,9 @@ def test_static_mounts_hash_cache_control(): ) incorrect_hash = hashlib.sha256(b"incorrect").hexdigest()[:12] - response = client.get(f"/custom-static/test_html.py?_hash={incorrect_hash}") + response = client.get( + "/custom-static/test_html.py?_hash={}".format(incorrect_hash) + ) assert response.status_code == 200 assert "cache-control" not in response.headers @@ -220,9 +219,11 @@ async def test_disallowed_custom_sql_pragma(ds_client): "/fixtures/-/query?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')" ) assert response.status_code == 400 - pragmas = ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas) + pragmas = ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas) assert ( - f"Statement contained a disallowed PRAGMA. Allowed pragma functions are {pragmas}" + "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format( + pragmas + ) in response.text ) @@ -777,8 +778,8 @@ def test_stored_query_show_hide_metadata_option( }, memory=True, ) as client: - expected_show_hide_fragment = ( - f'({expected_show_hide_text})' + expected_show_hide_fragment = '({})'.format( + expected_show_hide_link, expected_show_hide_text ) response = client.get("/_memory/one" + querystring) html = response.text @@ -787,7 +788,10 @@ def test_stored_query_show_hide_metadata_option( )[0] assert show_hide_fragment == expected_show_hide_fragment if expected_hidden: - assert f'' in html + assert ( + ''.format(expected_hidden) + in html + ) else: assert '; rel="alternate"; type="application/json+datasette"' + assert link == '<{}>; rel="alternate"; type="application/json+datasette"'.format( + expected + ) assert ( - f'' + ''.format( + expected + ) in response.text ) @@ -1284,8 +1292,8 @@ async def test_database_color(ds_client): expected_color = ds_client.ds.get_database("fixtures").color # Should be something like #9403e5 expected_fragments = ( - f"10px solid #{expected_color}", - f"border-color: #{expected_color}", + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), ) assert len(expected_color) == 6 for path in ( diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index b4bb964d..e1ab51bb 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,6 +1,5 @@ -import sqlite3 - import pytest +import sqlite3 from datasette.utils import escape_sqlite from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL @@ -138,7 +137,7 @@ async def test_internal_foreign_key_references(ds_client): return { row[1] for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" + "PRAGMA table_info({})".format(escape_sqlite(table_name)) ).fetchall() } @@ -148,7 +147,7 @@ async def test_internal_foreign_key_references(ds_client): for _, name in sorted( (row[5], row[1]) for row in conn.execute( - f"PRAGMA table_info({escape_sqlite(table_name)})" + "PRAGMA table_info({})".format(escape_sqlite(table_name)) ).fetchall() if row[5] ) @@ -160,7 +159,7 @@ async def test_internal_foreign_key_references(ds_client): for table_name in table_names: foreign_key_rows = conn.execute( - f"PRAGMA foreign_key_list({escape_sqlite(table_name)})" + "PRAGMA foreign_key_list({})".format(escape_sqlite(table_name)) ).fetchall() foreign_keys_by_id = {} for foreign_key in foreign_key_rows: @@ -170,16 +169,25 @@ async def test_internal_foreign_key_references(ds_client): foreign_key_rows.sort(key=lambda row: row[1]) other_table = foreign_key_rows[0][2] other_columns = [row[4] for row in foreign_key_rows] - message = f'Column "{table_name}.{foreign_key_rows[0][3]}" references other table "{other_table}" which does not exist' + message = 'Column "{}.{}" references other table "{}" which does not exist'.format( + table_name, foreign_key_rows[0][3], other_table + ) assert other_table in table_names, message + " (bad table)" if all(other_column is None for other_column in other_columns): other_columns = primary_keys_for_table(other_table) - length_message = f'Foreign key from "{table_name}" to "{other_table}" has {len(foreign_key_rows)} columns but references {len(other_columns)} columns' + length_message = 'Foreign key from "{}" to "{}" has {} columns but references {} columns'.format( + table_name, + other_table, + len(foreign_key_rows), + len(other_columns), + ) assert len(other_columns) == len(foreign_key_rows), length_message for foreign_key, other_column in zip(foreign_key_rows, other_columns): column = foreign_key[3] - message = f'Column "{table_name}.{column}" references other column "{other_table}.{other_column}" which does not exist' + message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format( + table_name, column, other_table, other_column + ) assert other_column in columns_by_table[other_table], ( message + " (bad column)" ) @@ -237,10 +245,10 @@ async def test_stale_catalog_entry_database_fix(tmp_path): @pytest.mark.asyncio async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") bravo_db_path = str(tmp_path / "bravo.db") @@ -285,10 +293,10 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path @pytest.mark.asyncio async def test_orphan_stale_catalog_child_entries_removed(tmp_path): - import sqlite3 - from datasette.app import Datasette + import sqlite3 + internal_db_path = str(tmp_path / "internal.db") alpha_db_path = str(tmp_path / "alpha.db") diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index b1093b1c..b1a212d9 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -3,23 +3,17 @@ Tests for the datasette.database.Database class """ import asyncio -import uuid from types import SimpleNamespace - +from datasette.app import Datasette +from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues +from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result +from datasette.utils.sqlite import sqlite3, supports_returning +from datasette.utils import Column import pytest import sqlite_utils - -from datasette.app import Datasette -from datasette.database import ( - Database, - DatasetteClosedError, - ExecuteWriteResult, - MultipleValues, - Results, - _deliver_write_result, -) -from datasette.utils import Column -from datasette.utils.sqlite import sqlite3, supports_returning +import time +import uuid requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -50,7 +44,7 @@ async def test_results_first(db): @pytest.mark.parametrize("expected", (True, False)) async def test_results_bool(db, expected): where = "" if expected else "where pk = 0" - results = await db.execute(f"select * from facetable {where}") + results = await db.execute("select * from facetable {}".format(where)) assert bool(results) is expected @@ -622,7 +616,7 @@ async def test_execute_write_block_false(db): "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], ) - await asyncio.sleep(0.1) + time.sleep(0.1) rows = await db.execute("select name from roadside_attractions where pk = 1") assert "Mystery!" == rows.rows[0][0] @@ -640,7 +634,7 @@ async def test_execute_write_with_returning_block_false(db): ) assert isinstance(task_id, uuid.UUID) - await asyncio.sleep(0.1) + time.sleep(0.1) assert ( await db.execute("select name from write_returning_block_false") ).single_value() == "Cleo" @@ -766,10 +760,9 @@ async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): # Plugins historically relied on the fact that the callback was invoked # positionally, so any parameter name worked. Preserve that contract. scope = {} - # exec() is how we build a function with a parameterized argument name - exec( # noqa: S102 - f"def write_fn({param_name}):\n" - f" return {param_name}.execute('select 1 + 1').fetchone()[0]", + exec( + "def write_fn({0}):\n" + " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), scope, ) write_fn = scope["write_fn"] @@ -793,9 +786,7 @@ async def test_execute_write_fn_with_track_event(db): @pytest.mark.asyncio -# func_only so the budget covers the write-thread call under test, not the -# one-off app_client fixture setup this test may be first to trigger -@pytest.mark.timeout(1, func_only=True) +@pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") conn = sqlite3.connect(path) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index ed2aeaf0..85598c05 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -9,15 +9,13 @@ import importlib import os import sqlite3 import time - -import pytest -from itsdangerous import BadSignature - from datasette import Context -from datasette.app import Database, Datasette, ResourcesSQL +from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from datasette.utils import PrefixedUrlString +from itsdangerous import BadSignature +import pytest @pytest.fixture @@ -79,7 +77,9 @@ async def test_static_template_function_hashes_core_asset(tmp_path, monkeypatch) template = ds.get_jinja_environment().from_string("{{ static('demo.js') }}") expected_hash = hashlib.sha256(b"const demo = true;").hexdigest()[:12] - assert await template.render_async() == f"/-/static/demo.js?_hash={expected_hash}" + assert await template.render_async() == "/-/static/demo.js?_hash={}".format( + expected_hash + ) assert isinstance(ds.static("demo.js"), PrefixedUrlString) @@ -101,7 +101,7 @@ def test_static_hash_recalculated_when_cache_headers_disabled(tmp_path, monkeypa asset_path.write_bytes(b"let a = 2;") expected_hash = hashlib.sha256(b"let a = 2;").hexdigest()[:12] - assert ds.static("demo.js") == f"/-/static/demo.js?_hash={expected_hash}" + assert ds.static("demo.js") == "/-/static/demo.js?_hash={}".format(expected_hash) assert ds.static("demo.js") != first_url @@ -114,12 +114,12 @@ def test_static_hashes_mounted_static_file(tmp_path): expected_hash = hashlib.sha256(b"body { color: black; }").hexdigest()[:12] assert ds.static("styles.css", mount="assets") == ( - f"/assets/styles.css?_hash={expected_hash}" + "/assets/styles.css?_hash={}".format(expected_hash) ) ds._settings["base_url"] = "/prefix/" assert ds.static("styles.css", mount="assets") == ( - f"/prefix/assets/styles.css?_hash={expected_hash}" + "/prefix/assets/styles.css?_hash={}".format(expected_hash) ) @@ -144,7 +144,9 @@ def test_static_hashes_plugin_static_file(tmp_path, monkeypatch): expected_hash = hashlib.sha256(b"console.log('plugin');").hexdigest()[:12] assert ds.static("plugin.js", plugin="datasette_cluster_map") == ( - f"/-/static-plugins/datasette_cluster_map/plugin.js?_hash={expected_hash}" + "/-/static-plugins/datasette_cluster_map/plugin.js?_hash={}".format( + expected_hash + ) ) diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 51b38f8d..e9aaaae8 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,7 +1,6 @@ import httpx import pytest import pytest_asyncio - from datasette.app import Datasette @@ -239,7 +238,7 @@ async def test_in_client_returns_false_outside_request(datasette): @pytest.mark.asyncio async def test_in_client_returns_true_inside_request(): """Test that datasette.in_client() returns True inside a client request""" - from datasette import Response, hookimpl + from datasette import hookimpl, Response class TestPlugin: __name__ = "test_in_client_plugin" diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index e982628b..6d2dc70a 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -1,8 +1,6 @@ -import json - -import pytest - from datasette.utils.asgi import PayloadTooLarge, Request +import json +import pytest def _post_scope(headers=None): diff --git a/tests/test_internals_response.py b/tests/test_internals_response.py index aa3e1ae2..2366dcde 100644 --- a/tests/test_internals_response.py +++ b/tests/test_internals_response.py @@ -1,8 +1,6 @@ -import json - -import pytest - from datasette.utils.asgi import Response +import json +import pytest def test_response_html(): diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 50c61995..24fa745d 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,7 +1,6 @@ -import pytest - from datasette.app import Datasette from datasette.utils import PrefixedUrlString +import pytest @pytest.fixture(scope="module") diff --git a/tests/test_label_column_for_table.py b/tests/test_label_column_for_table.py index b67b8882..7667b595 100644 --- a/tests/test_label_column_for_table.py +++ b/tests/test_label_column_for_table.py @@ -1,7 +1,6 @@ import pytest - -from datasette.app import Datasette from datasette.database import Database +from datasette.app import Datasette @pytest.mark.asyncio diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py index 61cdb3e0..cdadb091 100644 --- a/tests/test_load_extensions.py +++ b/tests/test_load_extensions.py @@ -1,8 +1,6 @@ -from pathlib import Path - -import pytest - from datasette.app import Datasette +import pytest +from pathlib import Path # not necessarily a full path - the full compiled path looks like "ext.dylib" # or another suffix, but sqlite will, under the hood, decide which file diff --git a/tests/test_messages.py b/tests/test_messages.py index 60eb7938..62d9f647 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -1,6 +1,5 @@ -import pytest - from .utils import cookie_was_deleted +import pytest @pytest.mark.asyncio diff --git a/tests/test_multipart.py b/tests/test_multipart.py index ab38bfb7..0dc3ecd7 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -6,12 +6,12 @@ Uses TDD approach - these tests are written first, then implementation follows. import base64 import json +import pytest from collections import namedtuple -import pytest from multipart_form_data_conformance import get_tests_dir -from datasette.utils.asgi import BadRequest, Request +from datasette.utils.asgi import Request, BadRequest def make_receive(body: bytes): diff --git a/tests/test_package.py b/tests/test_package.py index 43b20589..f05f3ece 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli class CaptureDockerfile: diff --git a/tests/test_permission_endpoints.py b/tests/test_permission_endpoints.py index c54bfbfd..8726ab62 100644 --- a/tests/test_permission_endpoints.py +++ b/tests/test_permission_endpoints.py @@ -6,7 +6,6 @@ Tests for permission endpoints: import pytest import pytest_asyncio - from datasette.app import Datasette @@ -433,8 +432,8 @@ async def test_execute_sql_requires_view_database(): A user who has execute-sql permission but not view-database permission should not be able to execute SQL on that database. """ - from datasette import hookimpl from datasette.permissions import PermissionSQL + from datasette import hookimpl class TestPermissionPlugin: __name__ = "TestPermissionPlugin" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 73c44682..cd1050d0 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,23 +1,20 @@ import collections -import copy -import json -import re -import time -import urllib -from pprint import pprint - -import pytest -import pytest_asyncio from asgiref.sync import async_to_sync -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner - from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action from datasette.utils import UNSTABLE_API_MESSAGE - from .fixtures import assert_permissions_checked, make_app_client +from click.testing import CliRunner +from bs4 import BeautifulSoup as Soup +import copy +import json +from pprint import pprint +import pytest_asyncio +import pytest +import re +import time +import urllib @pytest.fixture(scope="module") @@ -605,7 +602,9 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta ) assert ( response.status == expected_status - ), f"path: {path}, permissions: {permissions}, expected_status: {expected_status}, status: {response.status}" + ), "path: {}, permissions: {}, expected_status: {}, status: {}".format( + path, permissions, expected_status, response.status + ) finally: cascade_app_client.ds.config = previous_config @@ -2040,7 +2039,7 @@ async def test_databases_json_respects_view_database(tmp_path_factory): paths = [] for name in ("public", "private"): - path = str(db_directory / f"{name}.db") + path = str(db_directory / "{}.db".format(name)) conn = _sqlite3.connect(path) conn.execute("vacuum") conn.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 734f0fc2..59b1c0bf 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,3 +1,21 @@ +from bs4 import BeautifulSoup as Soup +from .fixtures import ( + make_app_client, + TEMP_PLUGIN_SECRET_FILE, + PLUGINS_DIR, + TestClient as _TestClient, +) # noqa +from click.testing import CliRunner +from datasette.app import Datasette +from datasette import cli, hookimpl +from datasette.fixtures import TABLES +from datasette.filters import FilterArguments +from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm +from datasette.permissions import PermissionSQL, Action +from datasette.resources import DatabaseResource +from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError, await_me_maybe +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime import importlib @@ -6,31 +24,8 @@ import os import pathlib import re import textwrap -import urllib - import pytest -from bs4 import BeautifulSoup as Soup -from click.testing import CliRunner -from jinja2 import ChoiceLoader, FileSystemLoader - -from datasette import cli, hookimpl -from datasette.app import Datasette -from datasette.filters import FilterArguments -from datasette.fixtures import TABLES -from datasette.permissions import Action, PermissionSQL -from datasette.plugins import DEFAULT_PLUGINS, get_plugins, pm -from datasette.resources import DatabaseResource -from datasette.utils import StartupError, await_me_maybe -from datasette.utils.sqlite import sqlite3 - -from .fixtures import ( - PLUGINS_DIR, - TEMP_PLUGIN_SECRET_FILE, - make_app_client, -) -from .fixtures import ( - TestClient as _TestClient, -) +import urllib at_memory_re = re.compile(r" at 0x\w+") @@ -40,7 +35,7 @@ at_memory_re = re.compile(r" at 0x\w+") ) def test_plugin_hooks_have_tests(plugin_hook): """Every plugin hook should be referenced in this test module""" - tests_in_this_module = [t for t in globals() if t.startswith("test_hook_")] + tests_in_this_module = [t for t in globals().keys() if t.startswith("test_hook_")] ok = False for test in tests_in_this_module: if plugin_hook in test: @@ -130,11 +125,11 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): response = await ds_client.get(path) assert response.status_code == 200 links = Soup(response.text, "html.parser").find_all("link") - special_href = next( + special_href = [ link for link in links if link.attrs["href"].endswith("/extra-css-urls-demo.css") - )["href"] + ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8")) @@ -157,7 +152,7 @@ async def test_hook_extra_js_urls(ds_client): "type": "module", }, ]: - assert any(s == attrs for s in script_attrs), f"Expected: {attrs}" + assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) @pytest.mark.asyncio @@ -320,8 +315,7 @@ async def test_plugin_config_env_from_list(ds_client): @pytest.mark.asyncio async def test_plugin_config_file(ds_client): - # Blocking write is fine here - it is tiny test setup, not request handling - with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: # noqa: ASYNC230 + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -829,7 +823,7 @@ def test_hook_register_routes_with_datasette(configured_path): assert response.status_code == 200 assert configured_path.upper() == response.text # Other one should 404 - other_path = next(p for p in ("path1", "path2") if configured_path != p) + other_path = [p for p in ("path1", "path2") if configured_path != p][0] assert client.get(f"/{other_path}/", follow_redirects=True).status_code == 404 @@ -934,7 +928,7 @@ async def test_plugin_startup_can_add_queries(): await datasette.add_query( "data", "from_startup", - f"select {result.first()[0]}", + "select {}".format(result.first()[0]), source="plugin", ) @@ -1046,7 +1040,7 @@ async def test_hook_handle_exception(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) async def test_hook_handle_exception_custom_response(ds_client, param): - response = await ds_client.get(f"/trigger-error?{param}=1") + response = await ds_client.get("/trigger-error?{}=1".format(param)) assert response.text == param @@ -1380,7 +1374,7 @@ async def test_hook_register_actions_no_duplicates(duplicate): # This should error: with pytest.raises(StartupError) as ex: await ds.invoke_startup() - assert f"Duplicate action {duplicate}" in str(ex.value) + assert "Duplicate action {}".format(duplicate) in str(ex.value) @pytest.mark.asyncio diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index aebcaa33..6617bc77 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -1,12 +1,10 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import json import os -import textwrap -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli +import textwrap @pytest.mark.serial @@ -72,7 +70,9 @@ def test_publish_cloudrun_prompts_for_service( ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} input-service --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format( + tag + ), shell=True, ), ] @@ -107,7 +107,9 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): ), mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} test --max-instances 1", + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format( + tag + ), shell=True, ), ] @@ -184,13 +186,13 @@ def test_publish_cloudrun_memory_cpu( tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test" expected_call = ( "gcloud run deploy --allow-unauthenticated --platform=managed" - f" --image {tag} test" + " --image {} test".format(tag) ) expected_build_call = f"gcloud builds submit --tag {tag}" if memory: - expected_call += f" --memory {memory}" + expected_call += " --memory {}".format(memory) if cpu: - expected_call += f" --cpu {cpu}" + expected_call += " --cpu {}".format(cpu) if timeout: expected_build_call += f" --timeout {timeout}" # max_instances defaults to 1 diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 4302ed94..cab83654 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,11 +1,9 @@ +from click.testing import CliRunner +from datasette import cli +from unittest import mock import os import pathlib -from unittest import mock - import pytest -from click.testing import CliRunner - -from datasette import cli @pytest.mark.serial diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 9b17d24b..3af1aace 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -20,7 +20,6 @@ def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: cwd=str(tmp_path), capture_output=True, text=True, - check=False, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index 15b7ad0f..ffa948a9 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,29 +15,37 @@ from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" ) -EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n id integer primary key,\n name text\n -- created text default (datetime('now'))\n)" +EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "\n".join( + ( + "create table new_table (", + " id integer primary key,", + " name text", + " -- created text default (datetime('now'))", + ")", + ) +) def _template_option_attributes(html, table): - match = re.search(rf'