diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 8112af24..3fc83438 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,46 +14,24 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - name: Check deployment prerequisites - id: deployment-prerequisites - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }} - run: | - missing=() - for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do - if [[ -z "${!variable:-}" ]]; then - missing+=("$variable") - fi - done - if (( ${#missing[@]} )); then - echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}" - echo "available=false" >> "$GITHUB_OUTPUT" - else - echo "available=true" >> "$GITHUB_OUTPUT" - fi - name: Check out datasette - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/checkout@v7 - name: Set up Python - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip - name: Install Python dependencies - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | python -m pip install --upgrade pip python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: | pytest -n auto -m "not serial" pytest -m "serial" - name: Build fixtures.db and other files needed to deploy the demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: |- python tests/fixtures.py \ fixtures.db \ @@ -62,14 +40,13 @@ jobs: plugins \ --extra-db-filename extra_database.db - name: Build docs.db - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} + if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | echo ' from datasette import hookimpl @@ -81,7 +58,6 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: And the counters writable stored query demo - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }} run: | cat > plugins/counters.py <=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET - - name: Upload latest documentation database to S3 (only for main) - if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - name: Deploy to docs as well (only for main) + if: ${{ github.ref == 'refs/heads/main' }} run: |- - # Keep development documentation separate from the stable release database. - s3-credentials put-object datasette-docs latest/docs.db docs.db \ - --content-type application/octet-stream + # Deploy docs.db to a different service + datasette publish cloudrun docs.db \ + --branch=$GITHUB_SHA \ + --version-note=$GITHUB_SHA \ + --extra-options="--setting template_debug 1" \ + --service=datasette-docs-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 232a34c7..21ed4c12 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -2,7 +2,7 @@ name: Publish Python Package on: release: - types: [published] + types: [created] permissions: contents: read @@ -51,8 +51,6 @@ jobs: - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 - # After the first non-prerelease 1.0 release, disable this job on 0.65.x, - # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs. deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] @@ -68,20 +66,26 @@ jobs: - name: Install dependencies run: | python -m pip install . --group dev - python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17" + python -m pip install sphinx-to-sqlite==0.1a1 - name: Build docs.db run: |- cd docs DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - - name: Upload stable documentation database to S3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }} + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + - name: Deploy stable-docs.datasette.io to Cloud Run run: |- - s3-credentials put-object datasette-docs docs.db docs.db \ - --content-type application/octet-stream + gcloud config set run/region us-central1 + gcloud config set project datasette-222320 + datasette publish cloudrun docs.db \ + --service=datasette-docs-stable deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a8c0ae4..751eedfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,17 +11,16 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) diff --git a/Dockerfile b/Dockerfile index 58287dd7..9a8f06cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim-bookworm AS build +FROM python:3.11.0-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 diff --git a/datasette/app.py b/datasette/app.py index 3251ed47..170c93ce 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -315,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child): actor_key = ( json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None ) - return (actor_key, action.name, parent, action.normalize_child(child)) + return (actor_key, action, parent, child) async def favicon(request, send): @@ -453,10 +453,8 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this - self._setup_db_done = False try: self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -464,7 +462,6 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() - self._startup_lock = asyncio.Lock() else: raise self.crossdb = crossdb @@ -1532,28 +1529,15 @@ class Datasette: conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") if self.sqlite_extensions and database != INTERNAL_DB_NAME: - # Extension loading is only enabled for as long as it takes to - # load the configured extensions. Leaving it enabled would let - # anyone who can execute SQL call load_extension() themselves. conn.enable_load_extension(True) - try: - for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. - if isinstance(extension, tuple): - path, entrypoint = extension - if sys.version_info >= (3, 12): - conn.load_extension(path, entrypoint=entrypoint) - else: - # Connection.load_extension() only gained the - # entrypoint argument in Python 3.12 - conn.execute( - "SELECT load_extension(?, ?)", [path, entrypoint] - ) - else: - conn.load_extension(extension) - finally: - conn.enable_load_extension(False) + for extension in self.sqlite_extensions: + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member @@ -1746,145 +1730,8 @@ class Datasette: sql, params = await build_allowed_resources_sql( self, actor, action, parent=parent, include_is_private=include_is_private ) - if action == "view-table": - sql, params = await self._apply_derived_table_permissions_to_sql( - sql, - params, - actor=actor, - parent=parent, - include_is_private=include_is_private, - ) return ResourcesSQL(sql, params) - async def _allowed_derived_table_source( - self, database, source, *, actor, dependencies - ): - """Check an immediate source, denying sources that are themselves derived.""" - if any( - TableResource.normalize_child(table) - == TableResource.normalize_child(source) - for table in dependencies - ): - return False - # The source has no dependency in this map. Evaluate its own permission - # and prerequisites without starting another dependency check. - verdicts = await self._allowed_many( - actions=["view-table"], - resource=TableResource(database, source), - actor=actor, - check_derived=False, - ) - return verdicts["view-table"] - - async def _apply_derived_table_permissions_to_sql( - self, - sql, - params, - *, - actor, - parent, - include_is_private, - ): - databases = ( - [(parent, self.databases[parent])] - if parent in self.databases - else ([] if parent is not None else list(self.databases.items())) - ) - dependency_maps = dict( - zip( - (name for name, _ in databases), - await asyncio.gather( - *(db.derived_table_dependencies() for _, db in databases) - ), - ) - ) - dependencies = [ - (database_name, child, source) - for database_name, dependency_map in dependency_maps.items() - for child, source in dependency_map.items() - ] - if not dependencies: - return sql, params - - sources = sorted( - {(database_name, source) for database_name, _, source in dependencies} - ) - actor_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=actor, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - actor_allowed = dict(zip(sources, actor_verdicts)) - - anonymous_allowed = {} - if include_is_private: - anonymous_verdicts = await asyncio.gather( - *( - self._allowed_derived_table_source( - database_name, - source, - actor=None, - dependencies=dependency_maps[database_name], - ) - for database_name, source in sources - ) - ) - anonymous_allowed = dict(zip(sources, anonymous_verdicts)) - - wrapped_params = dict(params) - derived_rows = [ - [ - database_name, - child, - int(actor_allowed[(database_name, source)]), - *( - [int(anonymous_allowed[(database_name, source)])] - if include_is_private - else [] - ), - ] - for database_name, child, source in dependencies - ] - derived_param = "_datasette_derived_permissions" - while derived_param in wrapped_params: - derived_param += "_" - wrapped_params[derived_param] = json.dumps(derived_rows) - - derived_columns = "parent, child, source_allowed" - select_columns = "allowed.parent, allowed.child, allowed.reason" - if include_is_private: - derived_columns += ", source_anonymous_allowed" - select_columns += ( - ", CASE WHEN derived.source_anonymous_allowed = 0 " - "THEN 1 ELSE allowed.is_private END AS is_private" - ) - wrapped_sql = f""" -WITH derived_permissions({derived_columns}) AS ( - SELECT - json_extract(value, '$[0]'), - json_extract(value, '$[1]'), - json_extract(value, '$[2]') - {", json_extract(value, '$[3]')" if include_is_private else ""} - FROM json_each(:{derived_param}) -), -allowed AS ( -{sql} -) -SELECT {select_columns} -FROM allowed -LEFT JOIN derived_permissions AS derived - ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE -WHERE COALESCE(derived.source_allowed, 1) = 1 -ORDER BY allowed.parent, allowed.child -""".strip() - return wrapped_sql, wrapped_params - async def allowed_resources( self, action: str, @@ -2087,12 +1934,6 @@ ORDER BY allowed.parent, allowed.child ) # {"edit-schema": True, "drop-table": True, "insert-row": False} """ - return await self._allowed_many( - actions=actions, resource=resource, actor=actor, check_derived=True - ) - - async def _allowed_many(self, *, actions, resource, actor, check_derived): - """Evaluate permissions, optionally applying the one-hop source policy.""" from datasette.permissions import ( _permission_check_cache, _skip_permission_checks, @@ -2130,7 +1971,7 @@ ORDER BY allowed.parent, allowed.child to_check = [] for name in expanded: if cache is not None: - key = _permission_cache_key(actor, self.actions[name], parent, child) + key = _permission_cache_key(actor, name, parent, child) if key in cache: final[name] = cache[key] continue @@ -2146,28 +1987,6 @@ ORDER BY allowed.parent, allowed.child child=child, ) - if ( - check_derived - and "view-table" in to_check - and raw.get("view-table") - and isinstance(resource, TableResource) - and parent in self.databases - ): - dependencies = await self.databases[parent].derived_table_dependencies() - source = next( - ( - source - for table, source in dependencies.items() - if TableResource.normalize_child(table) - == TableResource.normalize_child(child) - ), - None, - ) - if source is not None: - raw["view-table"] = await self._allowed_derived_table_source( - parent, source, actor=actor, dependencies=dependencies - ) - def resolve(name): # final verdict = own rules AND verdict of also_requires chain if name in final: @@ -2185,9 +2004,7 @@ ORDER BY allowed.parent, allowed.child # Cache the freshly computed checks if cache is not None: for name in to_check: - cache[ - _permission_cache_key(actor, self.actions[name], parent, child) - ] = final[name] + cache[_permission_cache_key(actor, name, parent, child)] = final[name] # Log every check (including cache hits) for the debug page, # dependencies before the actions that required them @@ -2629,7 +2446,7 @@ ORDER BY allowed.parent, allowed.child ): data = {"a": actor} if expire_after: - expires_at = int(time.time()) + expire_after + expires_at = int(time.time()) + (24 * 60 * 60) data["e"] = baseconv.base62.encode(expires_at) response.set_cookie("ds_actor", self.sign(data, "actor")) @@ -2986,52 +2803,24 @@ ORDER BY allowed.parent, allowed.child raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) - async def _startup_sequence(self): - """Idempotently run the full startup sequence: table counts for - immutable databases, then invoke_startup(). Safe to call more than - once and safe to call concurrently - callers block until whichever - call got there first has finished. - - This is the single entry point used by both AsgiLifespan (so - real deployments finish startup before accepting requests) and - AsgiRunOnFirstRequest (the fallback for hosts that never send - lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and - `datasette serve` (cli.py) calls it too. The fast path below checks - both `_startup_invoked` and `_setup_db_done` - not just the former - - so that a bare `await ds.invoke_startup()` made by a caller ahead of - `_startup_sequence()` (which only sets `_startup_invoked`) can't - make this method skip the immutable-database table-count precompute. - """ - if self._startup_invoked and self._setup_db_done: - return - async with self._startup_lock: - if self._startup_invoked and self._setup_db_done: - return - if not self._setup_db_done: - # First time server starts up, calculate table counts for - # immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - self._setup_db_done = True - await self.invoke_startup() - def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() + async def setup_db(): + # First time server starts up, calculate table counts for immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + async def _close_on_shutdown(): self.close() asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan( - asgi, - on_startup=[self._startup_sequence], - on_shutdown=[_close_on_shutdown], - ) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence]) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) return asgi @@ -3071,50 +2860,6 @@ class DatasetteRouter: receive, max_post_body_bytes=self.ds.setting("max_post_body_bytes"), ) - match, view = resolve_routes(self.routes, path) - is_static = view is favicon or getattr(view, "_datasette_static", False) - original_send = send - - async def send(message): - if message["type"] == "http.response.start" and not ( - is_static and message["status"] in (200, 304) - ): - # Decide privacy after rendering, including for streaming responses - # and error handlers. A public primary resource can still include - # private labels, actor navigation, or cookie-dependent content. - headers = list(message.get("headers", [])) - personalized = ( - request.actor is not None - or "cookie" in request.headers - or "authorization" in request.headers - or any(key.lower() == b"set-cookie" for key, _ in headers) - ) - if personalized: - headers = [ - (key, value) - for key, value in headers - if key.lower() != b"cache-control" - ] - headers.append((b"cache-control", b"private, no-store")) - - # Anonymous responses must not be reused for credentialed requests. - # Preserve any additional variation specified by views or plugins. - vary = [ - part.strip() - for key, value in headers - if key.lower() == b"vary" - for part in value.split(b",") - if part.strip() - ] - if b"*" not in vary: - for name in (b"Cookie", b"Authorization"): - if name.lower() not in {part.lower() for part in vary}: - vary.append(name) - headers = [(k, v) for k, v in headers if k.lower() != b"vary"] - headers.append((b"vary", b", ".join(vary))) - message = dict(message, headers=headers) - await original_send(message) - # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( @@ -3154,7 +2899,8 @@ class DatasetteRouter: return await self.handle_401(request, send, token_error) scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - request.scope = scope + + match, view = resolve_routes(self.routes, path) if match is None: return await self.handle_404(request, send) diff --git a/datasette/cli.py b/datasette/cli.py index 2694c1f6..57db83b6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -663,6 +663,16 @@ def serve( # Private utility mechanism for writing unit tests return ds + # Run async soundness checks before startup hooks, since invoke_startup + # now populates internal tables which requires querying each database + run_sync(lambda: check_databases(ds)) + + # Run the "startup" plugin hooks + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -670,14 +680,6 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: - # --get means we don't run Uvicorn at all - run_sync(lambda: check_databases(ds)) - - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - client = TestClient(ds) request_headers = {} if token: @@ -702,54 +704,34 @@ def serve( sys.exit(exit_code) return - # check_databases, invoke_startup() and the uvicorn server all run on a - # single event loop, so that anything a plugin's "startup" hook schedules - # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is - # still alive when the server starts handling requests. - async def _serve_async(): - # Populate internal catalog tables before invoke_startup - await check_databases(ds) - - # Run the full startup sequence (immutable-database table-count - # precompute + the "startup" plugin hooks) via the same entry point - # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when - # uvicorn's lifespan.startup fires moments later. - try: - await ds._startup_sequence() - except StartupError as e: - raise click.ClickException(e.args[0]) - - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = await initial_path_for_datasette(ds) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) - await server.serve() - - asyncio.run(_serve_async()) + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = run_sync(lambda: initial_path_for_datasette(ds)) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + uvicorn.run(ds.app(), **uvicorn_kwargs) @cli.command() diff --git a/datasette/database.py b/datasette/database.py index d444cbbf..e162d34e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,7 @@ from .utils import ( table_columns, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names +from .utils.sqlite import sqlite_hidden_table_names connections = threading.local() @@ -85,7 +85,6 @@ class Database: self.cached_hash = None self.cached_size = None self._cached_table_counts = None - self._cached_derived_table_dependencies = None self._write_thread = None self._write_queue = None self._closed = False @@ -247,29 +246,17 @@ class Database: return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, transaction=True, - time_limit_ms=2000, ): self._check_not_closed() if returning_limit < 0: raise ValueError("returning_limit must be >= 0") - def execute_sql(conn): + def _inner(conn): cursor = conn.execute(sql, params or []) return ExecuteWriteResult.from_cursor( cursor, return_all=return_all, returning_limit=returning_limit ) - def _inner(conn): - try: - if time_limit_ms is None: - return execute_sql(conn) - with sqlite_timelimit(conn, time_limit_ms): - return execute_sql(conn) - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - raise - with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn( _inner, block=block, request=request, transaction=transaction @@ -367,15 +354,6 @@ class Database: result = fn(self._write_connection) else: result = fn(self._write_connection) - if not block: - # There is no write thread here, so the write has already - # finished. Hand back the same (task_id, reply_future) shape - # _send_to_write_thread() returns, with the future already - # resolved, so the block=False path below is identical in - # both modes. - reply_future = asyncio.get_running_loop().create_future() - reply_future.set_result(result) - result = (uuid.uuid4(), reply_future) else: result = await self._send_to_write_thread( fn, block=block, transaction=transaction @@ -447,7 +425,7 @@ class Database: ) self._write_thread.name = f"_execute_writes for database {self.name}" self._write_thread.start() - task_id = uuid.uuid4() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() self._write_queue.put( @@ -781,17 +759,6 @@ class Database: return hidden_tables - async def derived_table_dependencies(self): - """Return implementation tables and the tables they derive from.""" - schema_version = (await self.execute("PRAGMA schema_version")).first()[0] - if ( - self._cached_derived_table_dependencies is None - or self._cached_derived_table_dependencies[0] != schema_version - ): - dependencies = await self.execute_fn(sqlite_derived_table_dependencies) - self._cached_derived_table_dependencies = (schema_version, dependencies) - return self._cached_derived_table_dependencies[1] - async def view_names(self): results = await self.execute("select name from sqlite_master where type='view'") return [r[0] for r in results.rows] diff --git a/datasette/default_column_types.py b/datasette/default_column_types.py index 6def3698..f90a733e 100644 --- a/datasette/default_column_types.py +++ b/datasette/default_column_types.py @@ -6,17 +6,6 @@ import markupsafe from datasette import hookimpl from datasette.column_types import ColumnType, SQLiteType -_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE) - - -def _normalize_http_url(value): - if not isinstance(value, str): - return None - normalized = value.strip() - if not _HTTP_URL_RE.fullmatch(normalized): - return None - return normalized - class UrlColumnType(ColumnType): name = "url" @@ -26,10 +15,7 @@ class UrlColumnType(ColumnType): async def render_cell(self, value, column, table, database, datasette, request): if not value or not isinstance(value, str): return None - normalized = _normalize_http_url(value) - if normalized is None: - return markupsafe.escape(value.strip()) - escaped = markupsafe.escape(normalized) + escaped = markupsafe.escape(value.strip()) return markupsafe.Markup(f'{escaped}') async def validate(self, value, datasette): @@ -37,7 +23,7 @@ class UrlColumnType(ColumnType): return None if not isinstance(value, str): return "URL must be a string" - if _normalize_http_url(value) is None: + if not re.match(r"^https?://\S+$", value.strip()): return "Invalid URL" return None diff --git a/datasette/default_permissions/config.py b/datasette/default_permissions/config.py index a4f5a4de..4494f07f 100644 --- a/datasette/default_permissions/config.py +++ b/datasette/default_permissions/config.py @@ -92,13 +92,6 @@ class ConfigPermissionProcessor: # Tables implicitly reference their parent databases self.restricted_databases.update(db for db, _ in self.restricted_tables) - # Resolve identity keys once per action, rather than scanning the - # restriction allowlist for every configured table's allow block. - self.restricted_table_keys = { - (db, self.action_obj.normalize_child(table) if self.action_obj else table) - for db, table in self.restricted_tables - } - def evaluate_allow_block(self, allow_block: Any) -> bool | None: """Evaluate an allow block against the current actor.""" if allow_block is None: @@ -132,10 +125,8 @@ class ConfigPermissionProcessor: if parent: table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {}) if child: - child_key = ( - self.action_obj.normalize_child(child) if self.action_obj else child - ) - if (parent, child_key) in self.restricted_table_keys: + table_actions = table_restrictions.get(child, []) + if self.action_checks.intersection(table_actions): return True else: # Parent query should proceed if any child in this database is allowlisted diff --git a/datasette/default_permissions/restrictions.py b/datasette/default_permissions/restrictions.py index d30ebd3f..88e1d274 100644 --- a/datasette/default_permissions/restrictions.py +++ b/datasette/default_permissions/restrictions.py @@ -185,15 +185,11 @@ def restrictions_allow_action( # Check table/resource level if resource is not None and not isinstance(resource, str) and len(resource) == 2: database, table = resource - action_obj = datasette.actions.get(action) - normalize = action_obj.normalize_child if action_obj else lambda name: name - for table_name, table_allowed in ( - restrictions.get("r", {}).get(database, {}).items() - ): - if normalize(table_name) == normalize(table): - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return True + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True # This action is not explicitly allowed, so reject it return False diff --git a/datasette/default_permissions/sqlite_statistics.py b/datasette/default_permissions/sqlite_statistics.py deleted file mode 100644 index 11fd4008..00000000 --- a/datasette/default_permissions/sqlite_statistics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Default table-access policy for SQLite optimizer statistics.""" - -import json - -from datasette import hookimpl -from datasette.permissions import PermissionSQL - - -@hookimpl -def permission_resources_sql(action): - if action != "view-table": - return None - return PermissionSQL( - sql=""" - SELECT database_name AS parent, value AS child, 0 AS allow, - 'SQLite statistics tables are denied by default' AS reason - FROM catalog_databases - CROSS JOIN json_each(:sqlite_statistics_names) - """, - params={ - "sqlite_statistics_names": json.dumps( - ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"] - ) - }, - ) diff --git a/datasette/filters.py b/datasette/filters.py index 83e51165..3cfb36e5 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -2,7 +2,7 @@ import json from typing import ClassVar from datasette import hookimpl -from datasette.resources import DatabaseResource, TableResource +from datasette.resources import DatabaseResource from datasette.utils.asgi import BadRequest from datasette.views.base import DatasetteError @@ -51,20 +51,13 @@ def search_filters(request, database, table, datasette): human_descriptions = [] extra_context = {} - # Figure out which trusted fts_table to use. Query string parameters can - # repeat this mapping (for backwards compatibility), but must not select - # a different table or primary key. + # Figure out which fts_table to use table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) - fts_table = table_metadata.get("fts_table") + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) - fts_pk = table_metadata.get("fts_pk", "rowid") - requested_fts_table = request.args.get("_fts_table") - requested_fts_pk = request.args.get("_fts_pk") - if (requested_fts_table and requested_fts_table != fts_table) or ( - requested_fts_pk and requested_fts_pk != fts_pk - ): - raise BadRequest("Invalid _fts_table or _fts_pk") + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = { key: request.args[key] for key in request.args @@ -82,11 +75,6 @@ def search_filters(request, database, table, datasette): extra_context["supports_search"] = bool(fts_table) if fts_table and search_args: - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=fts_table), - actor=request.actor, - ) if "_search" in search_args: # Simple ?_search=xxx search = search_args["_search"] @@ -147,11 +135,6 @@ def through_filters(request, database, table, datasette): through_table = through_data["table"] other_column = through_data["column"] value = through_data["value"] - await datasette.ensure_permission( - action="view-table", - resource=TableResource(database=database, table=through_table), - actor=request.actor, - ) db = datasette.get_database(database) outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) fk_to_us = next( diff --git a/datasette/permissions.py b/datasette/permissions.py index 2d242560..e03b065c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -3,10 +3,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any, NamedTuple -_SQLITE_IDENTIFIER_CASE = str.maketrans( - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" -) - # Context variable to track when permission checks should be skipped _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False @@ -53,15 +49,6 @@ class Resource(ABC): # Class-level metadata (subclasses must define these) name: str = None # e.g., "table", "database", "model" parent_class: type["Resource"] | None = None # e.g., DatabaseResource for tables - case_insensitive_child: bool = False - - @classmethod - def normalize_child(cls, child: str | None) -> str | None: - """Return a comparison key without changing the resource's display name.""" - if cls.case_insensitive_child and child is not None: - # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold. - return child.translate(_SQLITE_IDENTIFIER_CASE) - return child # Instance-level optional extra attributes reasons: list[str] | None = None @@ -159,11 +146,6 @@ class Action: resource_class: type[Resource] | None = None also_requires: str | None = None # Optional action name that must also be allowed - def normalize_child(self, child: str | None) -> str | None: - if self.resource_class is None: - return child - return self.resource_class.normalize_child(child) - @property def takes_parent(self) -> bool: """ diff --git a/datasette/plugins.py b/datasette/plugins.py index 6a4d7da7..9cf94079 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -18,7 +18,6 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_permissions.tokens", - "datasette.default_permissions.sqlite_statistics", "datasette.default_actions", "datasette.default_column_types", "datasette.default_magic_parameters", diff --git a/datasette/resources.py b/datasette/resources.py index 29bf7b1e..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -25,7 +25,6 @@ class TableResource(Resource): name = "table" parent_class = DatabaseResource - case_insensitive_child = True def __init__(self, database: str, table: str): super().__init__(parent=database, child=table) diff --git a/datasette/static/column-chooser.js b/datasette/static/column-chooser.js index c3d5796c..198641f3 100644 --- a/datasette/static/column-chooser.js +++ b/datasette/static/column-chooser.js @@ -472,13 +472,11 @@ class ColumnChooser extends HTMLElement { - + ${col}
`; - li.querySelector(".drag-item-label").textContent = col; - li.querySelector("input").addEventListener("change", (e) => { e.target.checked ? this._checked.add(col) : this._checked.delete(col); this._updateCounts(); diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..0e6e2c29 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,56 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === "object" && typeof module !== "undefined" + ? (module.exports = factory()) + : typeof define === "function" && define.amd + ? define(factory) + : (global.jsonFormatHighlight = factory()); +})(this, function () { + "use strict"; + + var defaultColors = { + keyColor: "dimgray", + numberColor: "lightskyblue", + stringColor: "lightcoral", + trueColor: "lightseagreen", + falseColor: "#f66578", + nullColor: "cornflowerblue", + }; + + function index(json, colorOptions) { + if (colorOptions === void 0) colorOptions = {}; + + if (!json) { + return; + } + if (typeof json !== "string") { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, "&").replace(//g, ">"); + return json.replace( + /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, + function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) + ? colors.trueColor + : /false/.test(match) + ? colors.falseColor + : /null/.test(match) + ? colors.nullColor + : color; + } + return '' + match + ""; + }, + ); + } + + return index; +}); diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 32686af1..4927cb8d 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -3,6 +3,7 @@ {% block title %}API Explorer{% endblock %} {% block extra_head %} + {% endblock %} {% block content %} @@ -125,7 +126,7 @@ getForm.addEventListener("submit", (ev) => { document.getElementById('response-status').textContent = response.status; return response.json(); }).then((data) => { - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); errorList.style.display = 'none'; }).catch((error) => { alert(error); @@ -173,7 +174,7 @@ postForm.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').textContent = JSON.stringify(data, null, 2); + output.querySelector('pre').innerHTML = jsonFormatHighlight(data); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/templates/debug_allowed.html b/datasette/templates/debug_allowed.html index c73cdfb7..80249d9c 100644 --- a/datasette/templates/debug_allowed.html +++ b/datasette/templates/debug_allowed.html @@ -3,6 +3,7 @@ {% block title %}Allowed Resources{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %} {% endblock %} @@ -197,7 +198,7 @@ function displayResults(data) { } // Update raw JSON - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } function displayError(data) { @@ -207,7 +208,7 @@ function displayError(data) { resultsContent.innerHTML = `
Error: ${escapeHtml(data.error || 'Unknown error')}
`; - document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2); + document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data); } // Disable child input if parent is empty diff --git a/datasette/templates/debug_check.html b/datasette/templates/debug_check.html index c0081c66..b9fc636a 100644 --- a/datasette/templates/debug_check.html +++ b/datasette/templates/debug_check.html @@ -3,6 +3,7 @@ {% block title %}Explain a permission decision{% endblock %} {% block extra_head %} + {% include "_permission_ui_styles.html" %} {% include "_debug_common_functions.html" %}