From 699be7dea9499355d576c16e330b85b90e1aca2a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 10:39:13 -0700 Subject: [PATCH 0001/1866] raise_404() function for use in custom templates, closes #964 --- datasette/app.py | 24 ++++++++--- docs/custom_templates.rst | 85 +++++++++++++++++++++++++++++--------- tests/test_custom_pages.py | 15 ++++++- 3 files changed, 97 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 17ae28ac..b17517ba 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1053,19 +1053,27 @@ class DatasetteRouter: headers["Location"] = location return "" + def raise_404(message=""): + raise NotFoundExplicit(message) + context.update( { "custom_header": custom_header, "custom_status": custom_status, "custom_redirect": custom_redirect, + "raise_404": raise_404, } ) - body = await self.ds.render_template( - template, - context, - request=request, - view_name="page", - ) + try: + body = await self.ds.render_template( + template, + context, + request=request, + view_name="page", + ) + except NotFoundExplicit as e: + await self.handle_500(request, send, e) + return # Pull content-type out into separate parameter content_type = "text/html; charset=utf-8" matches = [k for k in headers if k.lower() == "content-type"] @@ -1199,3 +1207,7 @@ def route_pattern_from_filepath(filepath): else: re_bits.append(re.escape(bit)) return re.compile("".join(re_bits)) + + +class NotFoundExplicit(NotFound): + pass diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 32dd6657..382fd8f3 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -1,7 +1,7 @@ .. _customization: -Customization -============= +Custom pages and templates +========================== Datasette provides a number of ways of customizing the way data is displayed. @@ -12,7 +12,9 @@ When you launch Datasette, you can specify a custom metadata file like this:: datasette mydb.db --metadata metadata.json -Your ``metadata.json`` file can include links that look like this:: +Your ``metadata.json`` file can include links that look like this: + +.. code-block:: json { "extra_css_urls": [ @@ -25,7 +27,9 @@ Your ``metadata.json`` file can include links that look like this:: The extra CSS and JavaScript files will be linked in the ```` of every page. -You can also specify a SRI (subresource integrity hash) for these assets:: +You can also specify a SRI (subresource integrity hash) for these assets: + +.. code-block:: json { "extra_css_urls": [ @@ -51,27 +55,39 @@ CSS classes on the Every default template includes CSS classes in the body designed to support custom styling. -The index template (the top level page at ``/``) gets this:: +The index template (the top level page at ``/``) gets this: + +.. code-block:: html -The database template (``/dbname``) gets this:: +The database template (``/dbname``) gets this: + +.. code-block:: html -The custom SQL template (``/dbname?sql=...``) gets this:: +The custom SQL template (``/dbname?sql=...``) gets this: + +.. code-block:: html -A canned query template (``/dbname/queryname``) gets this:: +A canned query template (``/dbname/queryname``) gets this: + +.. code-block:: html -The table template (``/dbname/tablename``) gets:: +The table template (``/dbname/tablename``) gets: + +.. code-block:: html -The row template (``/dbname/tablename/rowid``) gets:: +The row template (``/dbname/tablename/rowid``) gets: + +.. code-block:: html @@ -92,7 +108,9 @@ Some examples:: "no $ characters" => "no--characters-59e024" ```` and ```` elements also get custom CSS classes reflecting the -database column they are representing, for example:: +database column they are representing, for example: + +.. code-block:: html @@ -131,7 +149,9 @@ The following URLs will now serve the content from those CSS and JS files:: http://localhost:8001/static/styles.css http://localhost:8001/static/app.js -You can reference those files from ``metadata.json`` like so:: +You can reference those files from ``metadata.json`` like so: + +.. code-block:: json { "extra_css_urls": [ @@ -229,7 +249,9 @@ used that - but if that template had not been found, it would have tried for It is possible to extend the default templates using Jinja template inheritance. If you want to customize EVERY row template with some additional -content you can do so by creating a ``row.html`` template like this:: +content you can do so by creating a ``row.html`` template like this: + +.. code-block:: jinja {% extends "default:row.html" %} @@ -258,7 +280,9 @@ of a specific column. If you want to output the rendered HTML version of a column, including any links to foreign keys, you can use ``{{ row.display("column_name") }}``. -Here is an example of a custom ``_table.html`` template:: +Here is an example of a custom ``_table.html`` template: + +.. code-block:: jinja {% for row in display_rows %}
@@ -294,7 +318,7 @@ For example, to capture any request to a URL matching ``/about/*``, you would cr A hit to ``/about/news`` would render that template and pass in a variable called ``slug`` with a value of ``"news"``. -If you use this mechanism don't forget to return a 404 status code if the page should not be considered a valid page. You can do this using ``{{ custom_status(404) }}`` described below. +If you use this mechanism don't forget to return a 404 if the referenced content could not be found. You can do this using ``{{ raise_404() }}`` described below. Templates defined using custom page routes work particularly well with the ``sql()`` template function from `datasette-template-sql `__ or the ``graphql()`` template function from `datasette-graphql `__. @@ -305,7 +329,9 @@ Custom headers and status codes Custom pages default to being served with a content-type of ``text/html; charset=utf-8`` and a ``200`` status code. You can change these by calling a custom function from within your template. -For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code, create a file in ``pages/teapot.html`` containing the following:: +For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code, create a file in ``pages/teapot.html`` containing the following: + +.. code-block:: jinja {{ custom_status(418) }} @@ -315,7 +341,9 @@ For example, to serve a custom page with a ``418 I'm a teapot`` HTTP status code -To serve a custom HTTP header, add a ``custom_header(name, value)`` function call. For example:: +To serve a custom HTTP header, add a ``custom_header(name, value)`` function call. For example: + +.. code-block:: jinja {{ custom_status(418) }} {{ custom_header("x-teapot", "I am") }} @@ -335,17 +363,36 @@ You can verify this is working using ``curl`` like this:: x-teapot: I am content-type: text/html; charset=utf-8 +.. _custom_pages_404: + +Returning 404s +~~~~~~~~~~~~~~ + +To indicate that content could not be found and display the default 404 page you can use the ``raise_404(message)`` function: + +.. code-block:: jinja + + {% if not rows %} + {{ raise_404("Content not found") }} + {% endif %} + +If you call ``raise_404()`` the other content in your template will be ignored. + .. _custom_pages_redirects: Custom redirects ~~~~~~~~~~~~~~~~ -You can use the ``custom_redirect(location)`` function to redirect users to another page, for example in a file called ``pages/datasette.html``:: +You can use the ``custom_redirect(location)`` function to redirect users to another page, for example in a file called ``pages/datasette.html``: + +.. code-block:: jinja {{ custom_redirect("https://github.com/simonw/datasette") }} Now requests to ``http://localhost:8001/datasette`` will result in a redirect. -These redirects are served with a ``301 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function:: +These redirects are served with a ``301 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function: + +.. code-block:: jinja {{ custom_redirect("https://github.com/simonw/datasette", 301) }} diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index dc3be844..89e59a75 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -26,7 +26,11 @@ def custom_pages_client(tmp_path_factory): '{{ custom_redirect("/example", 301) }}', "utf-8" ) (pages_dir / "route_{name}.html").write_text( - "

Hello from {{ name }}

", "utf-8" + """ + {% if name == "OhNo" %}{{ raise_404("Oh no") }}{% endif %} +

Hello from {{ name }}

+ """, + "utf-8", ) nested_dir = pages_dir / "nested" nested_dir.mkdir() @@ -91,4 +95,11 @@ def test_redirect2(custom_pages_client): def test_custom_route_pattern(custom_pages_client): response = custom_pages_client.get("/route_Sally") assert response.status == 200 - assert response.text == "

Hello from Sally

" + assert response.text.strip() == "

Hello from Sally

" + + +def test_custom_route_pattern_404(custom_pages_client): + response = custom_pages_client.get("/route_OhNo") + assert response.status == 404 + assert "

Error 404

" in response.text + assert ">Oh no Date: Mon, 14 Sep 2020 11:30:31 -0700 Subject: [PATCH 0002/1866] Rename default error template to error.html, refs #965 --- datasette/app.py | 4 +--- datasette/templates/{500.html => error.html} | 0 2 files changed, 1 insertion(+), 3 deletions(-) rename datasette/templates/{500.html => error.html} (100%) diff --git a/datasette/app.py b/datasette/app.py index b17517ba..b39458dd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1124,9 +1124,7 @@ class DatasetteRouter: info = {} message = str(exception) traceback.print_exc() - templates = ["500.html"] - if status != 500: - templates = ["{}.html".format(status)] + templates + templates = ["{}.html".format(status), "error.html"] info.update( { "ok": False, diff --git a/datasette/templates/500.html b/datasette/templates/error.html similarity index 100% rename from datasette/templates/500.html rename to datasette/templates/error.html From 1552ac931e4d2cf516caac3ceeab4fd24da1510a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 11:47:16 -0700 Subject: [PATCH 0003/1866] Documented custom error pages, closes #965 --- docs/custom_templates.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 382fd8f3..d37bb729 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -396,3 +396,27 @@ These redirects are served with a ``301 Found`` status code by default. You can .. code-block:: jinja {{ custom_redirect("https://github.com/simonw/datasette", 301) }} + +.. _custom_pages_errors: + +Custom error pages +------------------ + +Datasette returns an error page if an unexpected error occurs, access is forbidden or content cannot be found. + +You can customize the response returned for these errors by providing a custom error page template. + +Content not found errors use a ``404.html`` template. Access denied errors use ``403.html``. Invalid input errors use ``400.html``. Unexpected errors of other kinds use ``500.html``. + +If a template for the specific error code is not found a template called ``error.html`` will be used instead. If you do not provide that template Datasette's `default error.html template `__ will be used. + +The error template will be passed the following context: + +``status`` - integer + The integer HTTP status code, e.g. 404, 500, 403, 400. + +``error`` - string + Details of the specific error, usually a full sentence. + +``title`` - string or None + A title for the page representing the class of error. This is often ``None`` for errors that do not provide a title separate from their ``error`` message. From 896fce228fec863354bd6267568c16ab13bb715a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 13:18:15 -0700 Subject: [PATCH 0004/1866] Canned query writes support JSON POST body, refs #880 --- datasette/utils/testing.py | 13 +++++++++---- datasette/views/database.py | 12 +++++++++++- tests/test_canned_queries.py | 11 +++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index dc261dc8..eb87fded 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -50,6 +50,7 @@ class TestClient: self, path, post_data=None, + body=None, allow_redirects=True, redirect_count=0, content_type="application/x-www-form-urlencoded", @@ -58,21 +59,25 @@ class TestClient: ): cookies = cookies or {} post_data = post_data or {} + assert not (post_data and body), "Provide one or other of body= or post_data=" # Maybe fetch a csrftoken first if csrftoken_from is not None: + assert body is None, "body= is not compatible with csrftoken_from=" if csrftoken_from is True: csrftoken_from = path token_response = await self._request(csrftoken_from, cookies=cookies) csrftoken = token_response.cookies["ds_csrftoken"] cookies["ds_csrftoken"] = csrftoken post_data["csrftoken"] = csrftoken + if post_data: + body = urlencode(post_data, doseq=True) return await self._request( path, allow_redirects, redirect_count, "POST", cookies, - post_data, + body, content_type, ) @@ -83,7 +88,7 @@ class TestClient: redirect_count=0, method="GET", cookies=None, - post_data=None, + post_body=None, content_type=None, ): query_string = b"" @@ -113,8 +118,8 @@ class TestClient: } instance = ApplicationCommunicator(self.asgi_app, scope) - if post_data: - body = urlencode(post_data, doseq=True).encode("utf-8") + if post_body: + body = post_body.encode("utf-8") await instance.send_input({"type": "http.request", "body": body}) else: await instance.send_input({"type": "http.request"}) diff --git a/datasette/views/database.py b/datasette/views/database.py index 556eaf24..62fa14c1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,6 +1,8 @@ import os import itertools import jinja2 +import json +from urllib.parse import parse_qsl from datasette.utils import ( check_visibility, @@ -208,7 +210,15 @@ class QueryView(DataView): # Execute query - as write or as read if write: if request.method == "POST": - params = await request.post_vars() + body = await request.post_body() + body = body.decode("utf-8").strip() + if body.startswith("{") and body.endswith("}"): + params = json.loads(body) + # But we want key=value strings + for key, value in params.items(): + params[key] = str(value) + else: + params = dict(parse_qsl(body, keep_blank_values=True)) if canned_query: params_for_query = MagicParameters(params, request, self.ds) else: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index fb9a0344..abf9c5bf 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -1,4 +1,5 @@ from bs4 import BeautifulSoup as Soup +import json import pytest import re from .fixtures import make_app_client, app_client @@ -163,6 +164,16 @@ def test_vary_header(canned_write_client): assert "Cookie" == canned_write_client.get("/data/update_name").headers["vary"] +def test_json_post_body(canned_write_client): + response = canned_write_client.post( + "/data/add_name", + body=json.dumps({"name": "Hello"}), + allow_redirects=False, + ) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + + def test_canned_query_permissions_on_database_page(canned_write_client): # Without auth only shows three queries query_names = { From 894999a14ef1b295673ef5f2063b043b2a15b769 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 13:25:09 -0700 Subject: [PATCH 0005/1866] Improved test for JSON POST, refs #880 --- tests/test_canned_queries.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index abf9c5bf..3b1f40bd 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -167,11 +167,13 @@ def test_vary_header(canned_write_client): def test_json_post_body(canned_write_client): response = canned_write_client.post( "/data/add_name", - body=json.dumps({"name": "Hello"}), + body=json.dumps({"name": ["Hello", "there"]}), allow_redirects=False, ) assert 302 == response.status assert "/data/add_name?success" == response.headers["Location"] + rows = canned_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] def test_canned_query_permissions_on_database_page(canned_write_client): From 72ac2fd32cb80a52fc4965872eb0146c3a3f99e3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 14:23:18 -0700 Subject: [PATCH 0006/1866] JSON API for writable canned queries, closes #880 --- datasette/utils/testing.py | 28 ++++++++++++++--------- datasette/views/database.py | 22 ++++++++++++++++-- docs/sql_queries.rst | 37 ++++++++++++++++++++++++++++++ tests/test_canned_queries.py | 44 +++++++++++++++++++++++++++++------- 4 files changed, 110 insertions(+), 21 deletions(-) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index eb87fded..6fc4c633 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -55,6 +55,7 @@ class TestClient: redirect_count=0, content_type="application/x-www-form-urlencoded", cookies=None, + headers=None, csrftoken_from=None, ): cookies = cookies or {} @@ -72,13 +73,14 @@ class TestClient: if post_data: body = urlencode(post_data, doseq=True) return await self._request( - path, - allow_redirects, - redirect_count, - "POST", - cookies, - body, - content_type, + path=path, + allow_redirects=allow_redirects, + redirect_count=redirect_count, + method="POST", + cookies=cookies, + headers=headers, + post_body=body, + content_type=content_type, ) async def _request( @@ -88,6 +90,7 @@ class TestClient: redirect_count=0, method="GET", cookies=None, + headers=None, post_body=None, content_type=None, ): @@ -99,14 +102,17 @@ class TestClient: raw_path = path.encode("latin-1") else: raw_path = quote(path, safe="/:,").encode("latin-1") - headers = [[b"host", b"localhost"]] + asgi_headers = [[b"host", b"localhost"]] + if headers: + for key, value in headers.items(): + asgi_headers.append([key.encode("utf-8"), value.encode("utf-8")]) if content_type: - headers.append((b"content-type", content_type.encode("utf-8"))) + asgi_headers.append((b"content-type", content_type.encode("utf-8"))) if cookies: sc = SimpleCookie() for key, value in cookies.items(): sc[key] = value - headers.append([b"cookie", sc.output(header="").encode("utf-8")]) + asgi_headers.append([b"cookie", sc.output(header="").encode("utf-8")]) scope = { "type": "http", "http_version": "1.0", @@ -114,7 +120,7 @@ class TestClient: "path": unquote(path), "raw_path": raw_path, "query_string": query_string, - "headers": headers, + "headers": asgi_headers, } instance = ApplicationCommunicator(self.asgi_app, scope) diff --git a/datasette/views/database.py b/datasette/views/database.py index 62fa14c1..8cd37fdb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -219,10 +219,17 @@ class QueryView(DataView): params[key] = str(value) else: params = dict(parse_qsl(body, keep_blank_values=True)) + # Should we return JSON? + should_return_json = ( + request.headers.get("accept") == "application/json" + or request.args.get("_json") + or params.get("_json") + ) if canned_query: params_for_query = MagicParameters(params, request, self.ds) else: params_for_query = params + ok = None try: cursor = await self.ds.databases[database].execute_write( sql, params_for_query, block=True @@ -234,12 +241,23 @@ class QueryView(DataView): ) message_type = self.ds.INFO redirect_url = metadata.get("on_success_redirect") + ok = True except Exception as e: message = metadata.get("on_error_message") or str(e) message_type = self.ds.ERROR redirect_url = metadata.get("on_error_redirect") - self.ds.add_message(request, message, message_type) - return self.redirect(request, redirect_url or request.path) + ok = False + if should_return_json: + return Response.json( + { + "ok": ok, + "message": message, + "redirect": redirect_url, + } + ) + else: + self.ds.add_message(request, message, message_type) + return self.redirect(request, redirect_url or request.path) else: async def extra_template(): diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index dd7743cf..1206db9b 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -326,6 +326,43 @@ The form presented at ``/mydatabase/add_message`` will have just a field for ``m Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. +.. _canned_queries_json_api: + +JSON API for writable canned queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Writable canned queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. + +To submit JSON to a writable canned query, encode key/value parameters as a JSON document:: + + POST /mydatabase/add_message + + {"message": "Message goes here"} + +You can also continue to submit data using regular form encoding, like so:: + + POST /mydatabase/add_message + + message=Message+goes+here + +There are three options for specifying that you would like the response to your request to return JSON data, as opposed to an HTTP redirect to another page. + +- Set an ``Accept: application/json`` header on your request +- Include ``?_json=1`` in the URL that you POST to +- Include ``"_json": 1`` in your JSON body, or ``&_json=1`` in your form encoded body + +The JSON response will look like this: + +.. code-block:: json + + { + "ok": true, + "message": "Query executed, 1 row affected", + "redirect": "/data/add_name" + } + +The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. + .. _pagination: Pagination diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 3b1f40bd..67e9f822 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -176,6 +176,33 @@ def test_json_post_body(canned_write_client): assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] +@pytest.mark.parametrize( + "headers,body,querystring", + ( + (None, "name=NameGoesHere", "?_json=1"), + ({"Accept": "application/json"}, "name=NameGoesHere", None), + (None, "name=NameGoesHere&_json=1", None), + (None, '{"name": "NameGoesHere", "_json": 1}', None), + ), +) +def test_json_response(canned_write_client, headers, body, querystring): + response = canned_write_client.post( + "/data/add_name" + (querystring or ""), + body=body, + allow_redirects=False, + headers=headers, + ) + assert 200 == response.status + assert response.headers["content-type"] == "application/json; charset=utf-8" + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": "/data/add_name?success", + } + rows = canned_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "NameGoesHere"}] + + def test_canned_query_permissions_on_database_page(canned_write_client): # Without auth only shows three queries query_names = { @@ -196,7 +223,14 @@ def test_canned_query_permissions_on_database_page(canned_write_client): cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, ) assert 200 == response.status - assert [ + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, {"name": "canned_read", "private": False}, @@ -204,13 +238,7 @@ def test_canned_query_permissions_on_database_page(canned_write_client): {"name": "from_async_hook", "private": False}, {"name": "from_hook", "private": False}, {"name": "update_name", "private": False}, - ] == sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) + ] def test_canned_query_permissions(canned_write_client): From c0249525d7df64471900f93ceb23e63463c46dd0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 14:38:24 -0700 Subject: [PATCH 0007/1866] Release 0.49 Refs #880, #944, #945, #947, #948, #953, #958, #962, #963, #964, #965 --- README.md | 1 + docs/changelog.rst | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 38ea7f79..08e19d4b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. * 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more. diff --git a/docs/changelog.rst b/docs/changelog.rst index 670cf558..3cd028e7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,22 +4,24 @@ Changelog ========= -.. _v0_49a1: +.. _v0_49: -0.49a1 (2020-09-13) -------------------- - -.. warning:: This is an **alpha** release. See :ref:`contributing_alpha_beta`. +0.49 (2020-09-14) +----------------- +- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (`#880 `__) +- New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (`#944 `__) - ``register_output_renderer()`` render functions can now return a ``Response``. (`#953 `__) - New ``--upgrade`` option for ``datasette install``. (`#945 `__) +- New ``datasette --pdb`` option. (`#962 `__) +- ``datasette --get`` exit code now reflects the internal HTTP status code. (`#947 `__) +- New ``raise_404()`` template function for returning 404 errors. (`#964 `__) - ``datasette publish heroku`` now deploys using Python 3.8.5 - Upgraded `CodeMirror `__ to 5.57.0. (`#948 `__) - Upgraded code style to Black 20.8b1. (`#958 `__) -- New ``datasette --pdb`` option. (`#962 `__) -- ``datasette --get`` exit code now reflects the internal HTTP status code. (`#947 `__) - Fixed bug where selected facets were not correctly persisted in hidden form fields on the table page. (`#963 `__) -- New mechanism for defining page templates with custom path parameters. (`#944 `__) +- Renamed the default error template from ``500.html`` to ``error.html``. +- Custom error pages are now documented, see :ref:`custom_pages_errors`. (`#965 `__) .. _v0_48: From 26de3a18bc47bdb152ab5909b2475a1ea202917b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 14:53:54 -0700 Subject: [PATCH 0008/1866] tmate debugging tool --- .github/workflows/tmate.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/tmate.yml diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml new file mode 100644 index 00000000..02e7bd33 --- /dev/null +++ b/.github/workflows/tmate.yml @@ -0,0 +1,12 @@ +name: tmate session + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 From cb515a9d75430adaf5e545a840bbc111648e8bfd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 15:09:03 -0700 Subject: [PATCH 0009/1866] Don't push preleases to Docker Hub, refs #940 --- .github/workflows/publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1db462a4..84a8be6c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -58,8 +58,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest needs: [deploy] - if: | - !(contains(github.ref, 'a') || contains(github.ref, 'b')) + if: "!github.event.release.prerelease" steps: - uses: actions/checkout@v2 - name: Build and push to Docker Hub From 853c5fc37011a7bc09ca3a1af287102f00827c82 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 14 Sep 2020 20:52:44 -0700 Subject: [PATCH 0010/1866] Fixed incorrect canned query example, closes #966 --- docs/sql_queries.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 1206db9b..0ce506cb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -316,9 +316,9 @@ Here's an example configuration (this time using ``metadata.yaml`` since that pr id: "*" sql: |- INSERT INTO messages ( - user_id, ip, message, datetime + user_id, message, datetime ) VALUES ( - :_actor_id, :_request_ip, :message, :_now_datetime_utc + :_actor_id, :message, :_now_datetime_utc ) write: true From 65ca17d729182277b8fbf81825ec90814ef24d6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2020 11:55:52 -0700 Subject: [PATCH 0011/1866] Fix for DeprecationWarning: invalid escape sequence --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index b39458dd..20aae7d0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1191,7 +1191,7 @@ def wrap_view(view_fn, datasette): return async_view_fn -_curly_re = re.compile("(\{.*?\})") +_curly_re = re.compile(r"(\{.*?\})") def route_pattern_from_filepath(filepath): From 448d13ea6b15a9c5c9165a4f6049d073f1524a5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2020 13:10:25 -0700 Subject: [PATCH 0012/1866] Fix for MagicParameters error with no POST body, closes #967 --- datasette/views/database.py | 5 +++++ tests/test_canned_queries.py | 37 +++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 8cd37fdb..c32ff92f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -371,6 +371,11 @@ class MagicParameters(dict): ) ) + def __len__(self): + # Workaround for 'Incorrect number of bindings' error + # https://github.com/simonw/datasette/issues/967#issuecomment-692951144 + return super().__len__() or 1 + def __getitem__(self, key): if key.startswith("_") and key.count("_") >= 2: prefix, suffix = key[1:].split("_", 1) diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 67e9f822..9620c693 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -305,14 +305,49 @@ def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re) assert None is soup.find("input", {"name": magic_parameter}) # Submit the form to create a log line response = magic_parameters_client.post( - "/data/runme_post", {}, csrftoken_from=True, cookies=cookies + "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies ) + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": None, + } post_actual = magic_parameters_client.get( "/data/logs.json?_sort_desc=rowid&_shape=array" ).json[0]["line"] assert re.match(expected_re, post_actual) +@pytest.mark.parametrize("use_csrf", [True, False]) +@pytest.mark.parametrize("return_json", [True, False]) +def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): + magic_parameters_client.ds._metadata["databases"]["data"]["queries"]["runme_post"][ + "sql" + ] = "insert into logs (line) values (:_header_host)" + qs = "" + if return_json: + qs = "?_json=1" + response = magic_parameters_client.post( + "/data/runme_post{}".format(qs), + {}, + csrftoken_from=use_csrf or None, + allow_redirects=False, + ) + if return_json: + assert response.status == 200 + assert response.json["ok"], response.json + else: + assert response.status == 302 + messages = magic_parameters_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Query executed, 1 row affected", 1]] == messages + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert post_actual == "localhost" + + def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): response = magic_parameters_client.get( "/data.json?sql=select+:_header_host&_shape=array" From d456b250326822b35169bbd52be5c143b6599fdc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2020 13:20:15 -0700 Subject: [PATCH 0013/1866] Release 0.49.1 Refs #967, #966, #956 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3cd028e7..a9fdd10c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_49_1: + +0.49.1 (2020-09-15) +------------------- + +- Fixed a bug with writable canned queries that use magic parameters but accept no non-magic arguments. (`#967 `__) + .. _v0_49: 0.49 (2020-09-14) From 432a3d675fa3697bd92fcc559d66e0c403d30c16 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2020 14:59:17 -0700 Subject: [PATCH 0014/1866] sqlite3.enable_callback_tracebacks(True), closes #891 --- datasette/utils/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cceff54b..bdae5653 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -26,6 +26,9 @@ try: except ImportError: import sqlite3 +if hasattr(sqlite3, "enable_callback_tracebacks"): + sqlite3.enable_callback_tracebacks(True) + # From https://www.sqlite.org/lang_keywords.html reserved_words = set( ( From 368be14c8bbb4cec607b89dcf677d089d743b649 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2020 17:01:11 -0700 Subject: [PATCH 0015/1866] Link to annotated release notes --- README.md | 2 +- docs/changelog.rst | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 08e19d4b..cc912a13 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News - * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. + * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. * 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more. diff --git a/docs/changelog.rst b/docs/changelog.rst index a9fdd10c..8bc88586 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,8 @@ Changelog 0.49 (2020-09-14) ----------------- +See also `Datasette 0.49: The annotated release notes `__. + - Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (`#880 `__) - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (`#944 `__) - ``register_output_renderer()`` render functions can now return a ``Response``. (`#953 `__) @@ -101,6 +103,8 @@ Changelog 0.45 (2020-07-01) ----------------- +See also `Datasette 0.45: The annotated release notes `__. + Magic parameters for canned queries, a log out feature, improved plugin documentation and four new plugin hooks. Magic parameters for canned queries @@ -157,6 +161,8 @@ Smaller changes 0.44 (2020-06-11) ----------------- +See also `Datasette 0.44: The annotated release notes `__. + Authentication and permissions, writable canned queries, flash messages, new plugin hooks and more. Authentication From a258339a935d8d29a95940ef1db01e98bb85ae63 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Sep 2020 23:33:09 -0700 Subject: [PATCH 0016/1866] Fixed typo --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8bc88586..685c4c13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -250,7 +250,7 @@ Datasette will generate a secret automatically when it starts up, but to avoid r You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`config_publish_secrets`. -Plugins can now sign value and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods. +Plugins can now sign values and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods. CSRF protection ~~~~~~~~~~~~~~~ From a980199e61fe7ccf02c2123849d86172d2ae54ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 07:26:47 -0700 Subject: [PATCH 0017/1866] New -o option for opening Datasette in your browser, closes #970 --- datasette/cli.py | 10 +++++++++- docs/datasette-serve-help.txt | 1 + docs/getting_started.rst | 4 ++++ tests/test_cli.py | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/cli.py b/datasette/cli.py index 231ae8b7..5d8333c6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -10,6 +10,7 @@ import shutil from subprocess import call import sys from runpy import run_module +import webbrowser from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm from .utils import ( check_connection, @@ -353,6 +354,7 @@ def uninstall(packages, yes): @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") +@click.option("-o", "--open", is_flag=True, help="Open Datasette in your web browser") def serve( files, immutable, @@ -375,6 +377,7 @@ def serve( version_note, help_config, pdb, + open, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -450,7 +453,12 @@ def serve( # Start the server if root: - print("http://{}:{}/-/auth-token?token={}".format(host, port, ds._root_token)) + url = "http://{}:{}/-/auth-token?token={}".format(host, port, ds._root_token) + print(url) + else: + url = "http://{}:{}/".format(host, port) + if open: + webbrowser.open(url) uvicorn.run(ds.app(), host=host, port=port, log_level="info", lifespan="on") diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 536e43b0..ac3ca49f 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -39,4 +39,5 @@ Options: --version-note TEXT Additional note to show on /-/versions --help-config Show available config options --pdb Launch debugger on any errors + -o, --open Open Datasette in your web browser --help Show this message and exit. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 44617b8b..2f0a7962 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -47,6 +47,10 @@ First, follow the :ref:`installation` instructions. Now you can run Datasette ag This will start a web server on port 8001 - visit http://localhost:8001/ to access the web interface. +Add ``-o`` to open your browser automatically once Datasette has started:: + + datasette path/to/database.db -o + Use Chrome on OS X? You can run datasette against your browser history like so: diff --git a/tests/test_cli.py b/tests/test_cli.py index b1c50282..093c84b5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -105,6 +105,7 @@ def test_metadata_yaml(): get=None, help_config=False, pdb=False, + open=False, return_instance=True, ) client = _TestClient(ds.app()) From cac051bb8aab88431c596d5ca321086df273164c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Sep 2020 08:37:59 -0700 Subject: [PATCH 0018/1866] Fix for 'open' bug, closes #973 --- datasette/cli.py | 12 +++++++++--- tests/test_cli.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 5d8333c6..28edd590 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -354,7 +354,13 @@ def uninstall(packages, yes): @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") -@click.option("-o", "--open", is_flag=True, help="Open Datasette in your web browser") +@click.option( + "-o", + "--open", + "open_browser", + is_flag=True, + help="Open Datasette in your web browser", +) def serve( files, immutable, @@ -377,7 +383,7 @@ def serve( version_note, help_config, pdb, - open, + open_browser, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -457,7 +463,7 @@ def serve( print(url) else: url = "http://{}:{}/".format(host, port) - if open: + if open_browser: webbrowser.open(url) uvicorn.run(ds.app(), host=host, port=port, log_level="info", lifespan="on") diff --git a/tests/test_cli.py b/tests/test_cli.py index 093c84b5..7ae9d6e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -105,7 +105,7 @@ def test_metadata_yaml(): get=None, help_config=False, pdb=False, - open=False, + open_browser=False, return_instance=True, ) client = _TestClient(ds.app()) From 9a6d0dce282e7fb58c5610e24c74098c923abfdc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Sep 2020 22:25:06 -0700 Subject: [PATCH 0019/1866] datasette-json-html as render_cell example --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index fc710a2b..fc023ad9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -398,7 +398,7 @@ If the value matches that pattern, the plugin returns an HTML link element: label=jinja2.escape(data["label"] or "") or " " )) -Examples: `datasette-render-binary `_, `datasette-render-markdown `_ +Examples: `datasette-render-binary `_, `datasette-render-markdown `__, `datasette-json-html `__ .. _plugin_register_output_renderer: From 1f021c37110fc9019b0ef70062c28c335e568ae2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 15:16:34 -0700 Subject: [PATCH 0020/1866] Update pytest requirement from <6.1.0,>=5.2.2 to >=5.2.2,<6.2.0 (#977) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...6.1.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7cf556c7..ddcd8106 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest>=5.2.2,<6.1.0", + "pytest>=5.2.2,<6.2.0", "pytest-asyncio>=0.10,<0.15", "beautifulsoup4>=4.8.1,<4.10.0", "black==20.8b1", From c11383e6284e000b2641569457efa16ac9e0d6ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Sep 2020 15:42:31 -0700 Subject: [PATCH 0021/1866] Fix rendering glitch with columns on mobile, closes #978 --- datasette/utils/__init__.py | 2 +- tests/test_utils.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bdae5653..c2578677 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -271,7 +271,7 @@ _boring_keyword_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") def escape_css_string(s): - return _css_re.sub(lambda m: "\\{:X}".format(ord(m.group())), s) + return _css_re.sub(lambda m: "\\" + ("{:X}".format(ord(m.group())).zfill(6)), s) def escape_sqlite(s): diff --git a/tests/test_utils.py b/tests/test_utils.py index 73001f0d..9f666386 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -420,6 +420,17 @@ def test_escape_fts(query, expected): assert expected == utils.escape_fts(query) +@pytest.mark.parametrize( + "input,expected", + [ + ("dog", "dog"), + ('dateutil_parse("1/2/2020")', r"dateutil_parse(\0000221/2/2020\000022)"), + ], +) +def test_escape_css_string(input, expected): + assert expected == utils.escape_css_string(input) + + def test_check_connection_spatialite_raises(): path = str(pathlib.Path(__file__).parent / "spatialite.db") conn = sqlite3.connect(path) From 5b8b8ae597cb5971b469923770e614b7fef30210 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Sep 2020 12:16:30 -0700 Subject: [PATCH 0022/1866] Handle \r\n correctly in CSS escapes, refs #980 --- datasette/utils/__init__.py | 5 ++++- tests/test_utils.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c2578677..0c310f6a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -271,7 +271,10 @@ _boring_keyword_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") def escape_css_string(s): - return _css_re.sub(lambda m: "\\" + ("{:X}".format(ord(m.group())).zfill(6)), s) + return _css_re.sub( + lambda m: "\\" + ("{:X}".format(ord(m.group())).zfill(6)), + s.replace("\r\n", "\n"), + ) def escape_sqlite(s): diff --git a/tests/test_utils.py b/tests/test_utils.py index 9f666386..c244a6f4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -425,6 +425,7 @@ def test_escape_fts(query, expected): [ ("dog", "dog"), ('dateutil_parse("1/2/2020")', r"dateutil_parse(\0000221/2/2020\000022)"), + ("this\r\nand\r\nthat", r"this\00000Aand\00000Athat"), ], ) def test_escape_css_string(input, expected): From ae1f7c3870001561392be85d2f4cd4d2aaf32d59 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Sep 2020 14:43:39 -0700 Subject: [PATCH 0023/1866] Column action menu for sort/faceting, refs #981 --- datasette/static/app.css | 47 +++++++++++++++++++++ datasette/static/table.js | 72 +++++++++++++++++++++++++++++++++ datasette/templates/_table.html | 2 +- datasette/templates/table.html | 1 + 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 datasette/static/table.js diff --git a/datasette/static/app.css b/datasette/static/app.css index 8428a933..034d355f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -32,6 +32,7 @@ td em { } th { padding-right: 1em; + white-space: nowrap; } table a:link { text-decoration: none; @@ -386,3 +387,49 @@ button.button-as-link { cursor: pointer; font-size: 1em; } + +.dropdown-menu { + display: inline-flex; + border: 1px solid #ccc; + border-radius: 4px; + line-height: 1.4; + font-size: 16px; + box-shadow: 2px 2px 2px #aaa; + background-color: #fff; +} +.dropdown-menu ul, +.dropdown-menu li { + list-style-type: none; + margin: 0; + padding: 0; +} +.dropdown-menu li { + border-bottom: 1px solid #ccc; +} +.dropdown-menu li:last-child { + border: none; +} +.dropdown-menu a:link, +.dropdown-menu a:visited, +.dropdown-menu a:hover, +.dropdown-menu a:focus +.dropdown-menu a:active { + text-decoration: none; + display: block; + padding: 4px 8px 2px 8px; + color: #222; +} +.dropdown-menu a:hover { + background-color: #eee; +} +.dropdown-menu .hook { + display: block; + position: absolute; + top: -5px; + left: 6px; + width: 0; + height: 0; + border-left: 5px solid transparent; + border-right: 5px solid transparent; + border-bottom: 5px solid #666; +} diff --git a/datasette/static/table.js b/datasette/static/table.js new file mode 100644 index 00000000..75cdd23c --- /dev/null +++ b/datasette/static/table.js @@ -0,0 +1,72 @@ +var DROPDOWN_HTML = `
+
+ +
`; + +var DROPDOWN_ICON_SVG = ` + + +`; + +(function() { + function sortDescUrl(column) { + return '?_sort_desc=' + encodeURIComponent(column); + } + function sortAscUrl(column) { + return '?_sort=' + encodeURIComponent(column); + } + function facetUrl(column) { + return '?_facet=' + encodeURIComponent(column); + } + + function iconClicked(ev) { + ev.preventDefault(); + var th = ev.target; + while (th.nodeName != 'TH') { + th = th.parentNode; + } + var rect = th.getBoundingClientRect(); + var menuTop = rect.bottom + window.scrollY; + var menuLeft = rect.left + window.scrollX; + var column = th.getAttribute('data-column'); + menu.querySelector('a.dropdown-sort-desc').setAttribute('href', sortDescUrl(column)); + menu.querySelector('a.dropdown-sort-asc').setAttribute('href', sortAscUrl(column)); + /* Only show facet if it's not the first column */ + var isFirstColumn = th.parentElement.querySelector('th:first-of-type') == th; + var facetItem = menu.querySelector('a.dropdown-facet'); + if (isFirstColumn) { + facetItem.style.display = 'none'; + } else { + facetItem.style.display = 'block'; + facetItem.setAttribute('href', facetUrl(column)); + } + menu.style.position = 'absolute'; + menu.style.top = (menuTop + 6) + 'px'; + menu.style.left = menuLeft + 'px'; + menu.style.display = 'block'; + } + var svg = document.createElement('div'); + svg.innerHTML = DROPDOWN_ICON_SVG; + svg = svg.querySelector('*'); + svg.style.display = 'inline-block'; + svg.style.position = 'relative'; + svg.style.top = '1px'; + var menu = document.createElement('div'); + menu.innerHTML = DROPDOWN_HTML; + menu.style.position = 'absolute'; + menu.style.display = 'none'; + menu = menu.querySelector('*'); + document.body.appendChild(menu); + + var ths = Array.from(document.querySelectorAll('.rows-and-columns th')); + ths.forEach(el => { + var icon = svg.cloneNode(true); + icon.addEventListener('click', iconClicked); + icon.style.cursor = 'pointer'; + el.appendChild(icon); + }); +})(); diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index 8fee77b2..1cb4110b 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -3,7 +3,7 @@
{% for column in display_columns %} - diff --git a/datasette/views/base.py b/datasette/views/base.py index 01e90220..a9953dfd 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -592,13 +592,15 @@ class DataView(BaseView): ) it_can_render = await await_me_maybe(it_can_render) if it_can_render: - renderers[key] = path_with_format( - request=request, format=key, extra_qs={**url_labels_extra} + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, format=key, extra_qs={**url_labels_extra} + ) ) url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = path_with_format( - request=request, format="csv", extra_qs=url_csv_args + url_csv = self.ds.urls.path( + path_with_format(request=request, format="csv", extra_qs=url_csv_args) ) url_csv_path = url_csv.split("?")[0] context = { diff --git a/datasette/views/database.py b/datasette/views/database.py index affded9b..f1901b34 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -459,7 +459,7 @@ class QueryView(DataView): "metadata": metadata, "settings": self.ds.settings_dict(), "request": request, - "show_hide_link": show_hide_link, + "show_hide_link": self.ds.urls.path(show_hide_link), "show_hide_text": show_hide_text, "show_hide_hidden": markupsafe.Markup(show_hide_hidden), "hide_sql": hide_sql, diff --git a/datasette/views/table.py b/datasette/views/table.py index e6ae67de..296e177f 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -942,6 +942,7 @@ class TableView(RowTableShared): "extra_wheres_for_ui": extra_wheres_for_ui, "form_hidden_args": form_hidden_args, "is_sortable": any(c["sortable"] for c in display_columns), + "fix_path": ds.urls.path, "path_with_replaced_args": path_with_replaced_args, "path_with_removed_args": path_with_removed_args, "append_querystring": append_querystring, diff --git a/tests/test_html.py b/tests/test_html.py index f24165bd..3301b91d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1614,11 +1614,16 @@ def test_metadata_sort_desc(app_client): "/fixtures/compound_three_primary_keys/a,a,a", "/fixtures/paginated_view", "/fixtures/facetable", + "/fixtures?sql=select+1", ], ) -def test_base_url_config(app_client_base_url_prefix, path): +@pytest.mark.parametrize("use_prefix", (True, False)) +def test_base_url_config(app_client_base_url_prefix, path, use_prefix): client = app_client_base_url_prefix - response = client.get("/prefix/" + path.lstrip("/")) + path_to_get = path + if use_prefix: + path_to_get = "/prefix/" + path.lstrip("/") + response = client.get(path_to_get) soup = Soup(response.body, "html.parser") for el in soup.findAll(["a", "link", "script"]): if "href" in el.attrs: @@ -1642,11 +1647,16 @@ def test_base_url_config(app_client_base_url_prefix, path): # If this has been made absolute it may start http://localhost/ if href.startswith("http://localhost/"): href = href[len("http://localost/") :] - assert href.startswith("/prefix/"), { - "path": path, - "href_or_src": href, - "element_parent": str(el.parent), - } + assert href.startswith("/prefix/"), json.dumps( + { + "path": path, + "path_to_get": path_to_get, + "href_or_src": href, + "element_parent": str(el.parent), + }, + indent=4, + default=repr, + ) def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): From 640031edfd40ba66aee3c4f7008c78c6a78a3e69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Nov 2021 17:01:17 -0800 Subject: [PATCH 0502/1866] Fixed bug introduced in #1519 --- datasette/views/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 296e177f..66447aa0 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -942,7 +942,7 @@ class TableView(RowTableShared): "extra_wheres_for_ui": extra_wheres_for_ui, "form_hidden_args": form_hidden_args, "is_sortable": any(c["sortable"] for c in display_columns), - "fix_path": ds.urls.path, + "fix_path": self.ds.urls.path, "path_with_replaced_args": path_with_replaced_args, "path_with_removed_args": path_with_removed_args, "append_querystring": append_querystring, From 24b5006ad7c316d00a1a963db5bfa82a49fab116 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 19 Nov 2021 17:11:13 -0800 Subject: [PATCH 0503/1866] ProxyPreserveHost On for apache-proxy demo, refs #1522 --- demos/apache-proxy/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 46697c63..59c20433 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -21,6 +21,7 @@ RUN echo -e 'ServerName localhost\n\ Allow from all\n\ \n\ \n\ +ProxyPreserveHost On\n\ ProxyPass /prefix/ http://localhost:8001/\n\ Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf From 494f11d5cc88f05df300f6f41bcf083a736487dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 10:51:14 -0800 Subject: [PATCH 0504/1866] Switch from Alpine to Debian, refs #1522 --- demos/apache-proxy/Dockerfile | 80 +++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 59c20433..40f5e31d 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -1,46 +1,64 @@ -FROM python:3-alpine +FROM python:3.9.7-slim-bullseye -RUN apk add --no-cache \ - apache2 \ - apache2-proxy \ - bash +RUN apt-get update && \ + apt-get install -y apache2 supervisor && \ + apt clean && \ + rm -rf /var/lib/apt && \ + rm -rf /var/lib/dpkg/info/* + +# Apache environment, copied from +# https://github.com/ijklim/laravel-benfords-law-app/blob/e9bf385dcaddb62ea466a7b245ab6e4ef708c313/docker/os/Dockerfile +ENV APACHE_DOCUMENT_ROOT=/var/www/html/public +ENV APACHE_RUN_USER www-data +ENV APACHE_RUN_GROUP www-data +ENV APACHE_PID_FILE /var/run/apache2.pid +ENV APACHE_RUN_DIR /var/run/apache2 +ENV APACHE_LOCK_DIR /var/lock/apache2 +ENV APACHE_LOG_DIR /var/log +RUN ln -sf /dev/stdout /var/log/apache2-access.log +RUN ln -sf /dev/stderr /var/log/apache2-error.log +RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR + +RUN a2enmod proxy +RUN a2enmod proxy_http +RUN a2enmod headers ARG DATASETTE_REF RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip -ENV TINI_VERSION v0.18.0 -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini -RUN chmod +x /tini - # Append this to the end of the default httpd.conf file -RUN echo -e 'ServerName localhost\n\ +RUN echo '\n\ +\n\ + Options Indexes FollowSymLinks\n\ + AllowOverride None\n\ + Require all granted\n\ +\n\ \n\ -\n\ - Order deny,allow\n\ - Allow from all\n\ -\n\ -\n\ -ProxyPreserveHost On\n\ -ProxyPass /prefix/ http://localhost:8001/\n\ -Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf - -RUN echo 'Datasette' > /var/www/localhost/htdocs/index.html +\n\ + ServerName localhost\n\ + DocumentRoot /app/html\n\ + ProxyPreserveHost On\n\ + ProxyPass /prefix/ http://127.0.0.1:8001/\n\ + Header add X-Proxied-By "Apache2 Debian"\n\ +\n\ +' > /etc/apache2/sites-enabled/000-default.conf WORKDIR /app +RUN mkdir -p /app/html +RUN echo 'Datasette' > /app/html/index.html ADD https://latest.datasette.io/fixtures.db /app/fixtures.db -RUN echo -e "#!/usr/bin/env bash\n\ -datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001 &\n\ -\n\ -httpd -D FOREGROUND & \n\ -\n\ -wait -n\n\ -exit $?" > /app/start.sh - -RUN chmod +x /app/start.sh - EXPOSE 80 -CMD /tini -- /app/start.sh +RUN echo "[supervisord]" >> /app/supervisord.conf +RUN echo "nodaemon=true" >> /app/supervisord.conf +RUN echo "" >> /app/supervisord.conf +RUN echo "[program:apache2]" >> /app/supervisord.conf +RUN echo "command=apache2 -D FOREGROUND" >> /app/supervisord.conf +RUN echo "" >> /app/supervisord.conf +RUN echo "[program:datasette]" >> /app/supervisord.conf +RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf + +CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"] From 48951e4304cc39b49e26682836d6961e165bddb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 10:51:51 -0800 Subject: [PATCH 0505/1866] Switch to hosting demo on Fly, closes #1522 --- demos/apache-proxy/README.md | 24 +++++++++++- .../{deploy.sh => deploy-to-cloud-run.sh} | 0 demos/apache-proxy/fly.toml | 37 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) rename demos/apache-proxy/{deploy.sh => deploy-to-cloud-run.sh} (100%) create mode 100644 demos/apache-proxy/fly.toml diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md index 08048512..c76e440d 100644 --- a/demos/apache-proxy/README.md +++ b/demos/apache-proxy/README.md @@ -2,7 +2,7 @@ See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy) -This live demo is running at https://apache-proxy-demo.datasette.io/ +This live demo is running at https://datasette-apache-proxy-demo.fly.dev/prefix/ To build locally, passing in a Datasette commit hash (or `main` for the main branch): @@ -14,3 +14,25 @@ Then run it like this: docker run -p 5000:80 datasette-apache-proxy-demo And visit `http://localhost:5000/` or `http://localhost:5000/prefix/` + +## Deployment to Fly + +To deploy to [Fly](https://fly.io/) first create an application there by running: + + flyctl apps create --name datasette-apache-proxy-demo + +You will need a different name, since I have already taken that one. + +Then run this command to deploy: + + flyctl deploy --build-arg DATASETTE_REF=main + +This uses `fly.toml` in this directory, which hard-codes the `datasette-apache-proxy-demo` name - so you would need to edit that file to match your application name before running this. + +## Deployment to Cloud Run + +Deployments to Cloud Run currently result in intermittent 503 errors and I'm not sure why, see [issue #1522](https://github.com/simonw/datasette/issues/1522). + +You can deploy like this: + + DATASETTE_REF=main ./deploy-to-cloud-run.sh diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy-to-cloud-run.sh similarity index 100% rename from demos/apache-proxy/deploy.sh rename to demos/apache-proxy/deploy-to-cloud-run.sh diff --git a/demos/apache-proxy/fly.toml b/demos/apache-proxy/fly.toml new file mode 100644 index 00000000..52e6af5d --- /dev/null +++ b/demos/apache-proxy/fly.toml @@ -0,0 +1,37 @@ +app = "datasette-apache-proxy-demo" + +kill_signal = "SIGINT" +kill_timeout = 5 +processes = [] + +[env] + +[experimental] + allowed_public_ports = [] + auto_rollback = true + +[[services]] + http_checks = [] + internal_port = 80 + processes = ["app"] + protocol = "tcp" + script_checks = [] + + [services.concurrency] + hard_limit = 25 + soft_limit = 20 + type = "connections" + + [[services.ports]] + handlers = ["http"] + port = 80 + + [[services.ports]] + handlers = ["tls", "http"] + port = 443 + + [[services.tcp_checks]] + grace_period = "1s" + interval = "15s" + restart_limit = 0 + timeout = "2s" From 08947fa76433d18988aa1ee1d929bd8320c75fe2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 11:03:08 -0800 Subject: [PATCH 0506/1866] Fix more broken base_url links Refs #1519, #838 --- datasette/facets.py | 10 +++++----- datasette/views/table.py | 2 +- tests/test_html.py | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 9a43b95e..29923ef7 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -225,8 +225,8 @@ class ColumnFacet(Facet): "name": column, "type": self.type, "hideable": source != "metadata", - "toggle_url": path_with_removed_args( - self.request, {"_facet": column} + "toggle_url": ds.urls.path( + path_with_removed_args(self.request, {"_facet": column}) ), "results": facet_results_values, "truncated": len(facet_rows_results) > facet_size, @@ -259,7 +259,7 @@ class ColumnFacet(Facet): "label": expanded.get((column, row["value"]), row["value"]), "count": row["count"], "toggle_url": self.ds.absolute_url( - self.request, toggle_path + self.request, self.ds.urls.path(toggle_path) ), "selected": selected, } @@ -397,8 +397,8 @@ class ArrayFacet(Facet): "type": self.type, "results": facet_results_values, "hideable": source != "metadata", - "toggle_url": path_with_removed_args( - self.request, {"_facet_array": column} + "toggle_url": self.ds.urls.path( + path_with_removed_args(self.request, {"_facet_array": column}) ), "truncated": len(facet_rows_results) > facet_size, } diff --git a/datasette/views/table.py b/datasette/views/table.py index 66447aa0..1960f455 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -839,7 +839,7 @@ class TableView(RowTableShared): else: added_args = {"_next": next_value} next_url = self.ds.absolute_url( - request, path_with_replaced_args(request, added_args) + request, self.ds.urls.path(path_with_replaced_args(request, added_args)) ) rows = rows[:page_size] diff --git a/tests/test_html.py b/tests/test_html.py index 3301b91d..68508d75 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1614,6 +1614,7 @@ def test_metadata_sort_desc(app_client): "/fixtures/compound_three_primary_keys/a,a,a", "/fixtures/paginated_view", "/fixtures/facetable", + "/fixtures/facetable?_facet=state", "/fixtures?sql=select+1", ], ) From 250db8192cb8aba5eb8cd301ccc2a49525bc3d24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 11:09:05 -0800 Subject: [PATCH 0507/1866] Hopefully last fix relating to #1519, #838 --- datasette/facets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/facets.py b/datasette/facets.py index 29923ef7..8fd2177a 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -225,7 +225,7 @@ class ColumnFacet(Facet): "name": column, "type": self.type, "hideable": source != "metadata", - "toggle_url": ds.urls.path( + "toggle_url": self.ds.urls.path( path_with_removed_args(self.request, {"_facet": column}) ), "results": facet_results_values, From f11a13d73f021906f04b495cd589915e9a926bc5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 12:23:40 -0800 Subject: [PATCH 0508/1866] Extract out Apache config to separate file, refs #1524 --- demos/apache-proxy/000-default.conf | 13 +++++++++++++ demos/apache-proxy/Dockerfile | 20 +++----------------- 2 files changed, 16 insertions(+), 17 deletions(-) create mode 100644 demos/apache-proxy/000-default.conf diff --git a/demos/apache-proxy/000-default.conf b/demos/apache-proxy/000-default.conf new file mode 100644 index 00000000..5b6607a3 --- /dev/null +++ b/demos/apache-proxy/000-default.conf @@ -0,0 +1,13 @@ + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + + + ServerName localhost + DocumentRoot /app/html + ProxyPreserveHost On + ProxyPass /prefix/ http://127.0.0.1:8001/ + Header add X-Proxied-By "Apache2 Debian" + diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 40f5e31d..0854b552 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -27,31 +27,17 @@ ARG DATASETTE_REF RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip -# Append this to the end of the default httpd.conf file -RUN echo '\n\ -\n\ - Options Indexes FollowSymLinks\n\ - AllowOverride None\n\ - Require all granted\n\ -\n\ -\n\ -\n\ - ServerName localhost\n\ - DocumentRoot /app/html\n\ - ProxyPreserveHost On\n\ - ProxyPass /prefix/ http://127.0.0.1:8001/\n\ - Header add X-Proxied-By "Apache2 Debian"\n\ -\n\ -' > /etc/apache2/sites-enabled/000-default.conf +ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf WORKDIR /app RUN mkdir -p /app/html -RUN echo 'Datasette' > /app/html/index.html +RUN echo '

Demo is at /prefix/

' > /app/html/index.html ADD https://latest.datasette.io/fixtures.db /app/fixtures.db EXPOSE 80 +# Dynamically build supervisord config since it includes $DATASETTE_REF: RUN echo "[supervisord]" >> /app/supervisord.conf RUN echo "nodaemon=true" >> /app/supervisord.conf RUN echo "" >> /app/supervisord.conf From ed77eda6d8f10c63fc0670c7150fc974f786ade5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 15:30:25 -0800 Subject: [PATCH 0509/1866] Add datasette-redirect-to-https plugin Also configured suprvisord children to log to stdout, so that I can see them with flyctly logs -a datasette-apache-proxy-demo Refs #1524 --- demos/apache-proxy/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 0854b552..ab7b9d16 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -25,7 +25,9 @@ RUN a2enmod headers ARG DATASETTE_REF -RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip +RUN pip install \ + https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip \ + datasette-redirect-to-https ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf @@ -43,8 +45,12 @@ RUN echo "nodaemon=true" >> /app/supervisord.conf RUN echo "" >> /app/supervisord.conf RUN echo "[program:apache2]" >> /app/supervisord.conf RUN echo "command=apache2 -D FOREGROUND" >> /app/supervisord.conf +RUN echo "stdout_logfile=/dev/stdout" >> /app/supervisord.conf +RUN echo "stdout_logfile_maxbytes=0" >> /app/supervisord.conf RUN echo "" >> /app/supervisord.conf RUN echo "[program:datasette]" >> /app/supervisord.conf RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf +RUN echo "stdout_logfile=/dev/stdout" >> /app/supervisord.conf +RUN echo "stdout_logfile_maxbytes=0" >> /app/supervisord.conf CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"] From d8c79b1340ceb742077587fb7f76ed8699d4e402 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 15:33:58 -0800 Subject: [PATCH 0510/1866] Link to Apache proxy demo from documentation, closes #1524 --- docs/deploying.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index 83d9e4dd..d4ad8836 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -188,6 +188,8 @@ Then add these directives to proxy traffic:: ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPreserveHost On +A live demo of Datasette running behind Apache using this proxy setup can be seen at `datasette-apache-proxy-demo.datasette.io/prefix/ `__. The code for that demo can be found in the `demos/apache-proxy `__ directory. + Using ``--uds`` you can use Unix domain sockets similar to the nginx example:: ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ From 48f11998b73350057b74fe6ab464d4ac3071637c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Nov 2021 15:40:21 -0800 Subject: [PATCH 0511/1866] Release 0.59.3 Refs #448, #838, #1519 --- datasette/version.py | 2 +- docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index db89b418..0ba55573 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.59.2" +__version__ = "0.59.3" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 47ca3480..449ce412 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_59_3: + +0.59.3 (2021-11-20) +------------------- + +- Fixed numerous bugs when running Datasette :ref:`behind a proxy ` with a prefix URL path using the :ref:`setting_base_url` setting. A live demo of this mode is now available at `datasette-apache-proxy-demo.datasette.io/prefix/ `__. (:issue:`1519`, :issue:`838`) +- ``?column__arraycontains=`` and ``?column__arraynotcontains=`` table parameters now also work against SQL views. (:issue:`448`) +- ``?_facet_array=column`` no longer returns incorrect counts if columns contain the same value more than once. + .. _v0_59_2: 0.59.2 (2021-11-13) From 1beb7d939999da79bb77c4d3c777657c8a16bcd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:29:54 -0800 Subject: [PATCH 0512/1866] Update aiofiles requirement from <0.8,>=0.4 to >=0.4,<0.9 (#1537) Updates the requirements on [aiofiles](https://github.com/Tinche/aiofiles) to permit the latest version. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.4.0...v0.8.0) --- updated-dependencies: - dependency-name: aiofiles dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17a56a97..9e205ce2 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ setup( "pint~=0.9", "pluggy>=0.13,<1.1", "uvicorn~=0.11", - "aiofiles>=0.4,<0.8", + "aiofiles>=0.4,<0.9", "janus>=0.6.2,<0.7", "asgi-csrf>=0.9", "PyYAML>=5.3,<7.0", From 3303514a52b7170f2f1e598cd9c5f82c22f26e6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:35:18 -0800 Subject: [PATCH 0513/1866] Update docutils requirement from <0.18 to <0.19 (#1508) Updates the requirements on [docutils](http://docutils.sourceforge.net/) to permit the latest version. --- updated-dependencies: - dependency-name: docutils dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/readthedocs-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt index 93120e66..db1851ad 100644 --- a/docs/readthedocs-requirements.txt +++ b/docs/readthedocs-requirements.txt @@ -1 +1 @@ -docutils<0.18 +docutils<0.19 From cc4c70b3670ce2a85bb883b8d5626574590efe14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:35:28 -0800 Subject: [PATCH 0514/1866] Bump black from 21.9b0 to 21.11b1 (#1516) Bumps [black](https://github.com/psf/black) from 21.9b0 to 21.11b1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9e205ce2..71422d87 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( "pytest-xdist>=2.2.1,<2.5", "pytest-asyncio>=0.10,<0.17", "beautifulsoup4>=4.8.1,<4.11.0", - "black==21.9b0", + "black==21.11b1", "pytest-timeout>=1.4.2,<2.1", "trustme>=0.7,<0.10", ], From 83eb29deced2430f40c3374ff9085d65d86d8281 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:37:13 -0800 Subject: [PATCH 0515/1866] Update janus requirement from <0.7,>=0.6.2 to >=0.6.2,<0.8 (#1529) Updates the requirements on [janus](https://github.com/aio-libs/janus) to permit the latest version. - [Release notes](https://github.com/aio-libs/janus/releases) - [Changelog](https://github.com/aio-libs/janus/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/janus/compare/v0.6.2...v0.7.0) --- updated-dependencies: - dependency-name: janus dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 71422d87..3cb657e3 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( "pluggy>=0.13,<1.1", "uvicorn~=0.11", "aiofiles>=0.4,<0.9", - "janus>=0.6.2,<0.7", + "janus>=0.6.2,<0.8", "asgi-csrf>=0.9", "PyYAML>=5.3,<7.0", "mergedeep>=1.1.1,<1.4.0", From 06762776f712526fdb40a18ed26f259be62bb214 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 19:04:20 -0800 Subject: [PATCH 0516/1866] Fix for incorrect hidden for fields for _columns, refs #1527 --- datasette/views/table.py | 6 +++++- tests/test_html.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1960f455..9fc6afcf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -889,7 +889,11 @@ class TableView(RowTableShared): form_hidden_args = [] for key in request.args: - if key.startswith("_") and key not in ("_sort", "_search", "_next"): + if ( + key.startswith("_") + and key not in ("_sort", "_search", "_next") + and not key.endswith("__exact") + ): for value in request.args.getlist(key): form_hidden_args.append((key, value)) diff --git a/tests/test_html.py b/tests/test_html.py index 68508d75..179c3f09 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -326,6 +326,19 @@ def test_existing_filter_redirects(app_client): assert "?" not in response.headers["Location"] +def test_exact_parameter_results_in_correct_hidden_fields(app_client): + # https://github.com/simonw/datasette/issues/1527 + response = app_client.get( + "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown" + ) + # In this case we should NOT have a hidden _neighborhood__exact=Downtown field + form = Soup(response.body, "html.parser").find("form") + hidden_inputs = { + input["name"]: input["value"] for input in form.select("input[type=hidden]") + } + assert hidden_inputs == {"_facet": "_neighborhood"} + + def test_empty_search_parameter_gets_removed(app_client): path_base = "/fixtures/simple_primary_key" path = ( From 69244a617b1118dcbd04a8f102173f04680cf08c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 22:17:27 -0800 Subject: [PATCH 0517/1866] Rename city_id to _city_id in fixtures, refs #1525 --- tests/fixtures.py | 8 ++--- tests/test_api.py | 50 ++++++++++++++++---------------- tests/test_facets.py | 36 +++++++++++------------ tests/test_html.py | 34 +++++++++++----------- tests/test_internals_database.py | 4 +-- tests/test_plugins.py | 8 ++--- 6 files changed, 70 insertions(+), 70 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1a879126..37399da0 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -358,7 +358,7 @@ METADATA = { select _neighborhood, facet_cities.name, state from facetable join facet_cities - on facetable.city_id = facet_cities.id + on facetable._city_id = facet_cities.id where _neighborhood like '%' || :text || '%' order by _neighborhood; """ @@ -558,15 +558,15 @@ CREATE TABLE facetable ( planet_int integer, on_earth integer, state text, - city_id integer, + _city_id integer, _neighborhood text, tags text, complex_array text, distinct_some_null, - FOREIGN KEY ("city_id") REFERENCES [facet_cities](id) + FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null) + (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null) VALUES ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'), ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'), diff --git a/tests/test_api.py b/tests/test_api.py index 43b52175..8b3fcd75 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -197,7 +197,7 @@ def test_database_page(app_client): { "other_table": "facetable", "column": "id", - "other_column": "city_id", + "other_column": "_city_id", } ], "outgoing": [], @@ -212,7 +212,7 @@ def test_database_page(app_client): "planet_int", "on_earth", "state", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", @@ -227,7 +227,7 @@ def test_database_page(app_client): "outgoing": [ { "other_table": "facet_cities", - "column": "city_id", + "column": "_city_id", "other_column": "id", } ], @@ -1512,40 +1512,40 @@ def test_page_size_matching_max_returned_rows( "path,expected_facet_results", [ ( - "/fixtures/facetable.json?_facet=state&_facet=city_id", + "/fixtures/facetable.json?_facet=state&_facet=_city_id", { "state": { "name": "state", "hideable": True, "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=city_id", + "toggle_url": "/fixtures/facetable.json?_facet=_city_id", "results": [ { "value": "CA", "label": "CA", "count": 10, - "toggle_url": "_facet=state&_facet=city_id&state=CA", + "toggle_url": "_facet=state&_facet=_city_id&state=CA", "selected": False, }, { "value": "MI", "label": "MI", "count": 4, - "toggle_url": "_facet=state&_facet=city_id&state=MI", + "toggle_url": "_facet=state&_facet=_city_id&state=MI", "selected": False, }, { "value": "MC", "label": "MC", "count": 1, - "toggle_url": "_facet=state&_facet=city_id&state=MC", + "toggle_url": "_facet=state&_facet=_city_id&state=MC", "selected": False, }, ], "truncated": False, }, - "city_id": { - "name": "city_id", + "_city_id": { + "name": "_city_id", "hideable": True, "type": "column", "toggle_url": "/fixtures/facetable.json?_facet=state", @@ -1554,28 +1554,28 @@ def test_page_size_matching_max_returned_rows( "value": 1, "label": "San Francisco", "count": 6, - "toggle_url": "_facet=state&_facet=city_id&city_id=1", + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1", "selected": False, }, { "value": 2, "label": "Los Angeles", "count": 4, - "toggle_url": "_facet=state&_facet=city_id&city_id=2", + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2", "selected": False, }, { "value": 3, "label": "Detroit", "count": 4, - "toggle_url": "_facet=state&_facet=city_id&city_id=3", + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3", "selected": False, }, { "value": 4, "label": "Memnonia", "count": 1, - "toggle_url": "_facet=state&_facet=city_id&city_id=4", + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4", "selected": False, }, ], @@ -1584,26 +1584,26 @@ def test_page_size_matching_max_returned_rows( }, ), ( - "/fixtures/facetable.json?_facet=state&_facet=city_id&state=MI", + "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI", { "state": { "name": "state", "hideable": True, "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=city_id&state=MI", + "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI", "results": [ { "value": "MI", "label": "MI", "count": 4, "selected": True, - "toggle_url": "_facet=state&_facet=city_id", + "toggle_url": "_facet=state&_facet=_city_id", } ], "truncated": False, }, - "city_id": { - "name": "city_id", + "_city_id": { + "name": "_city_id", "hideable": True, "type": "column", "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI", @@ -1613,7 +1613,7 @@ def test_page_size_matching_max_returned_rows( "label": "Detroit", "count": 4, "selected": False, - "toggle_url": "_facet=state&_facet=city_id&state=MI&city_id=3", + "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3", } ], "truncated": False, @@ -1699,7 +1699,7 @@ def test_suggested_facets(app_client): {"name": "planet_int", "querystring": "_facet=planet_int"}, {"name": "on_earth", "querystring": "_facet=on_earth"}, {"name": "state", "querystring": "_facet=state"}, - {"name": "city_id", "querystring": "_facet=city_id"}, + {"name": "_city_id", "querystring": "_facet=_city_id"}, {"name": "_neighborhood", "querystring": "_facet=_neighborhood"}, {"name": "tags", "querystring": "_facet=tags"}, {"name": "complex_array", "querystring": "_facet=complex_array"}, @@ -1765,7 +1765,7 @@ def test_expand_labels(app_client): "planet_int": 1, "on_earth": 1, "state": "CA", - "city_id": {"value": 1, "label": "San Francisco"}, + "_city_id": {"value": 1, "label": "San Francisco"}, "_neighborhood": "Dogpatch", "tags": '["tag1", "tag3"]', "complex_array": "[]", @@ -1777,7 +1777,7 @@ def test_expand_labels(app_client): "planet_int": 1, "on_earth": 1, "state": "MI", - "city_id": {"value": 3, "label": "Detroit"}, + "_city_id": {"value": 3, "label": "Detroit"}, "_neighborhood": "Corktown", "tags": "[]", "complex_array": "[]", @@ -2128,7 +2128,7 @@ def test_http_options_request(app_client): "planet_int", "on_earth", "state", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", @@ -2155,7 +2155,7 @@ def test_http_options_request(app_client): "created", "planet_int", "on_earth", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", diff --git a/tests/test_facets.py b/tests/test_facets.py index a20c79c4..429117cb 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -23,7 +23,7 @@ async def test_column_facet_suggest(app_client): {"name": "planet_int", "toggle_url": "http://localhost/?_facet=planet_int"}, {"name": "on_earth", "toggle_url": "http://localhost/?_facet=on_earth"}, {"name": "state", "toggle_url": "http://localhost/?_facet=state"}, - {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"}, + {"name": "_city_id", "toggle_url": "http://localhost/?_facet=_city_id"}, { "name": "_neighborhood", "toggle_url": "http://localhost/?_facet=_neighborhood", @@ -56,8 +56,8 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client): "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=state", }, { - "name": "city_id", - "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=city_id", + "name": "_city_id", + "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_city_id", }, { "name": "_neighborhood", @@ -82,7 +82,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["city_id"]}, + metadata={"facets": ["_city_id"]}, ) suggestions = [s["name"] for s in await facet.suggest()] assert [ @@ -100,7 +100,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): async def test_column_facet_results(app_client): facet = ColumnFacet( app_client.ds, - Request.fake("/?_facet=city_id"), + Request.fake("/?_facet=_city_id"), database="fixtures", sql="select * from facetable", table="facetable", @@ -108,8 +108,8 @@ async def test_column_facet_results(app_client): buckets, timed_out = await facet.facet_results() assert [] == timed_out assert { - "city_id": { - "name": "city_id", + "_city_id": { + "name": "_city_id", "type": "column", "hideable": True, "toggle_url": "/", @@ -118,28 +118,28 @@ async def test_column_facet_results(app_client): "value": 1, "label": "San Francisco", "count": 6, - "toggle_url": "http://localhost/?_facet=city_id&city_id=1", + "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=1", "selected": False, }, { "value": 2, "label": "Los Angeles", "count": 4, - "toggle_url": "http://localhost/?_facet=city_id&city_id=2", + "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=2", "selected": False, }, { "value": 3, "label": "Detroit", "count": 4, - "toggle_url": "http://localhost/?_facet=city_id&city_id=3", + "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=3", "selected": False, }, { "value": 4, "label": "Memnonia", "count": 1, - "toggle_url": "http://localhost/?_facet=city_id&city_id=4", + "toggle_url": "http://localhost/?_facet=_city_id&_city_id__exact=4", "selected": False, }, ], @@ -278,13 +278,13 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["city_id"]}, + metadata={"facets": ["_city_id"]}, ) buckets, timed_out = await facet.facet_results() assert [] == timed_out assert { - "city_id": { - "name": "city_id", + "_city_id": { + "name": "_city_id", "type": "column", "hideable": False, "toggle_url": "/", @@ -293,28 +293,28 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client): "value": 1, "label": "San Francisco", "count": 6, - "toggle_url": "http://localhost/?city_id=1", + "toggle_url": "http://localhost/?_city_id__exact=1", "selected": False, }, { "value": 2, "label": "Los Angeles", "count": 4, - "toggle_url": "http://localhost/?city_id=2", + "toggle_url": "http://localhost/?_city_id__exact=2", "selected": False, }, { "value": 3, "label": "Detroit", "count": 4, - "toggle_url": "http://localhost/?city_id=3", + "toggle_url": "http://localhost/?_city_id__exact=3", "selected": False, }, { "value": 4, "label": "Memnonia", "count": 1, - "toggle_url": "http://localhost/?city_id=4", + "toggle_url": "http://localhost/?_city_id__exact=4", "selected": False, }, ], diff --git a/tests/test_html.py b/tests/test_html.py index 179c3f09..aaf7da09 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -487,7 +487,7 @@ def test_sort_links(app_client): def test_facet_display(app_client): response = app_client.get( - "/fixtures/facetable?_facet=planet_int&_facet=city_id&_facet=on_earth" + "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth" ) assert response.status == 200 soup = Soup(response.body, "html.parser") @@ -509,26 +509,26 @@ def test_facet_display(app_client): ) assert actual == [ { - "name": "city_id", + "name": "_city_id", "items": [ { "name": "San Francisco", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=1", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1", "count": 6, }, { "name": "Los Angeles", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=2", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2", "count": 4, }, { "name": "Detroit", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=3", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3", "count": 4, }, { "name": "Memnonia", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&city_id=4", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4", "count": 1, }, ], @@ -538,12 +538,12 @@ def test_facet_display(app_client): "items": [ { "name": "1", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&planet_int=1", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1", "count": 14, }, { "name": "2", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&planet_int=2", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2", "count": 1, }, ], @@ -553,12 +553,12 @@ def test_facet_display(app_client): "items": [ { "name": "1", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&on_earth=1", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1", "count": 14, }, { "name": "0", - "qs": "_facet=planet_int&_facet=city_id&_facet=on_earth&on_earth=0", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0", "count": 1, }, ], @@ -568,14 +568,14 @@ def test_facet_display(app_client): def test_facets_persist_through_filter_form(app_client): response = app_client.get( - "/fixtures/facetable?_facet=planet_int&_facet=city_id&_facet_array=tags" + "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags" ) assert response.status == 200 inputs = Soup(response.body, "html.parser").find("form").findAll("input") hiddens = [i for i in inputs if i["type"] == "hidden"] assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [ ("_facet", "planet_int"), - ("_facet", "city_id"), + ("_facet", "_city_id"), ("_facet_array", "tags"), ] @@ -1350,20 +1350,20 @@ def test_canned_query_show_hide_metadata_option( def test_extra_where_clauses(app_client): response = app_client.get( - "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=city_id=1" + "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=_city_id=1" ) soup = Soup(response.body, "html.parser") div = soup.select(".extra-wheres")[0] assert "2 extra where clauses" == div.find("h3").text hrefs = [a["href"] for a in div.findAll("a")] assert [ - "/fixtures/facetable?_where=city_id%3D1", + "/fixtures/facetable?_where=_city_id%3D1", "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27", ] == hrefs # These should also be persisted as hidden fields inputs = soup.find("form").findAll("input") hiddens = [i for i in inputs if i["type"] == "hidden"] - assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [ + assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "_city_id=1")] == [ (hidden["name"], hidden["value"]) for hidden in hiddens ] @@ -1683,11 +1683,11 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): [ ( "/fixtures/neighborhood_search", - "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=", + "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=", ), ( "/fixtures/neighborhood_search?text=ber", - "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber", + "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber", ), ("/fixtures/pragma_cache_size", None), ( diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 2d0cae7f..a00fe447 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -81,7 +81,7 @@ async def test_table_exists(db, tables, exists): "planet_int", "on_earth", "state", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", @@ -161,7 +161,7 @@ async def test_table_columns(db, table, expected): ), Column( cid=5, - name="city_id", + name="_city_id", type="integer", notnull=0, default_value=None, diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 697a6b32..1da28453 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -436,7 +436,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): "planet_int", "on_earth", "state", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", @@ -459,7 +459,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): "", "", ], - "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", "query_name": None, "database": "fixtures", "table": "facetable", @@ -525,13 +525,13 @@ def test_hook_register_output_renderer_can_render(app_client): "planet_int", "on_earth", "state", - "city_id", + "_city_id", "_neighborhood", "tags", "complex_array", "distinct_some_null", ], - "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", "query_name": None, "database": "fixtures", "table": "facetable", From a37ee74891f14898d5810127c7ca3355e77ff57d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 22:34:31 -0800 Subject: [PATCH 0518/1866] Correct link to _ prefix on row page, closes #1525 --- datasette/templates/row.html | 2 +- datasette/views/table.py | 10 +++++++++- tests/test_html.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 916980b6..c86e979d 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -38,7 +38,7 @@
    {% for other in foreign_key_tables %}
  • - + {{ "{:,}".format(other.count) }} row{% if other.count == 1 %}{% else %}s{% endif %} from {{ other.other_column }} in {{ other.other_table }}
    • diff --git a/datasette/views/table.py b/datasette/views/table.py index 9fc6afcf..f58b78f5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1120,5 +1120,13 @@ class RowView(RowTableShared): count = ( foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0 ) - foreign_key_tables.append({**fk, **{"count": count}}) + key = fk["other_column"] + if key.startswith("_"): + key += "__exact" + link = "{}?{}={}".format( + self.ds.urls.table(database, fk["other_table"]), + key, + ",".join(pk_values), + ) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) return foreign_key_tables diff --git a/tests/test_html.py b/tests/test_html.py index aaf7da09..a7cb105c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -820,6 +820,34 @@ def test_row_html_no_primary_key(app_client): ] +@pytest.mark.parametrize( + "path,expected_text,expected_link", + ( + ( + "/fixtures/facet_cities/1", + "6 rows from _city_id in facetable", + "/fixtures/facetable?_city_id__exact=1", + ), + ( + "/fixtures/attraction_characteristic/2", + "3 rows from characteristic_id in roadside_attraction_characteristics", + "/fixtures/roadside_attraction_characteristics?characteristic_id=2", + ), + ), +) +def test_row_links_from_other_tables(app_client, path, expected_text, expected_link): + response = app_client.get(path) + assert response.status == 200 + soup = Soup(response.body, "html.parser") + h2 = soup.find("h2") + assert h2.text == "Links from other tables" + li = h2.findNext("ul").find("li") + text = re.sub(r"\s+", " ", li.text.strip()) + assert text == expected_text + link = li.find("a")["href"] + assert link == expected_link + + def test_table_html_compound_primary_key(app_client): response = app_client.get("/fixtures/compound_primary_key") assert response.status == 200 From 35b12746ba2bf9f254791bddac03d25b19be9b77 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 22:37:22 -0800 Subject: [PATCH 0519/1866] Fixed CSV test I broke in #1525 --- tests/test_csv.py | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index 5902e9db..8749cd8b 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -24,7 +24,7 @@ world ) EXPECTED_TABLE_WITH_LABELS_CSV = """ -pk,created,planet_int,on_earth,state,city_id,city_id_label,_neighborhood,tags,complex_array,distinct_some_null +pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null 1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one 2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two 3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[], @@ -57,42 +57,42 @@ def test_table_csv(app_client): response = app_client.get("/fixtures/simple_primary_key.csv?_oh=1") assert response.status == 200 assert not response.headers.get("Access-Control-Allow-Origin") - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_TABLE_CSV == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == EXPECTED_TABLE_CSV def test_table_csv_cors_headers(app_client_with_cors): response = app_client_with_cors.get("/fixtures/simple_primary_key.csv") assert response.status == 200 - assert "*" == response.headers["Access-Control-Allow-Origin"] + assert response.headers["Access-Control-Allow-Origin"] == "*" def test_table_csv_no_header(app_client): response = app_client.get("/fixtures/simple_primary_key.csv?_header=off") assert response.status == 200 assert not response.headers.get("Access-Control-Allow-Origin") - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_TABLE_CSV.split("\r\n", 1)[1] == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == EXPECTED_TABLE_CSV.split("\r\n", 1)[1] def test_table_csv_with_labels(app_client): response = app_client.get("/fixtures/facetable.csv?_labels=1") assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_TABLE_WITH_LABELS_CSV == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == EXPECTED_TABLE_WITH_LABELS_CSV def test_table_csv_with_nullable_labels(app_client): response = app_client.get("/fixtures/foreign_key_references.csv?_labels=1") assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV def test_table_csv_blob_columns(app_client): response = app_client.get("/fixtures/binary_data.csv") assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert response.headers["content-type"] == "text/plain; charset=utf-8" assert response.text == ( "rowid,data\r\n" "1,http://localhost/fixtures/binary_data/1.blob?_blob_column=data\r\n" @@ -104,7 +104,7 @@ def test_table_csv_blob_columns(app_client): def test_custom_sql_csv_blob_columns(app_client): response = app_client.get("/fixtures.csv?sql=select+rowid,+data+from+binary_data") assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert response.headers["content-type"] == "text/plain; charset=utf-8" assert response.text == ( "rowid,data\r\n" '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' @@ -118,16 +118,18 @@ def test_custom_sql_csv(app_client): "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2" ) assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_CUSTOM_CSV == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == EXPECTED_CUSTOM_CSV def test_table_csv_download(app_client): response = app_client.get("/fixtures/simple_primary_key.csv?_dl=1") assert response.status == 200 - assert "text/csv; charset=utf-8" == response.headers["content-type"] - expected_disposition = 'attachment; filename="simple_primary_key.csv"' - assert expected_disposition == response.headers["content-disposition"] + assert response.headers["content-type"] == "text/csv; charset=utf-8" + assert ( + response.headers["content-disposition"] + == 'attachment; filename="simple_primary_key.csv"' + ) def test_csv_with_non_ascii_characters(app_client): @@ -135,8 +137,8 @@ def test_csv_with_non_ascii_characters(app_client): "/fixtures.csv?sql=select%0D%0A++%27%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC%27+as+text%2C%0D%0A++1+as+number%0D%0Aunion%0D%0Aselect%0D%0A++%27bob%27+as+text%2C%0D%0A++2+as+number%0D%0Aorder+by%0D%0A++number" ) assert response.status == 200 - assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" == response.text + assert response.headers["content-type"] == "text/plain; charset=utf-8" + assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" def test_max_csv_mb(app_client_csv_max_mb_one): @@ -156,10 +158,10 @@ def test_max_csv_mb(app_client_csv_max_mb_one): def test_table_csv_stream(app_client): # Without _stream should return header + 100 rows: response = app_client.get("/fixtures/compound_three_primary_keys.csv?_size=max") - assert 101 == len([b for b in response.body.split(b"\r\n") if b]) + assert len([b for b in response.body.split(b"\r\n") if b]) == 101 # With _stream=1 should return header + 1001 rows response = app_client.get("/fixtures/compound_three_primary_keys.csv?_stream=1") - assert 1002 == len([b for b in response.body.split(b"\r\n") if b]) + assert len([b for b in response.body.split(b"\r\n") if b]) == 1002 def test_csv_trace(app_client_with_trace): From ca6624643842f4b80644b83c3f4ad7c2265c15d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 22:45:04 -0800 Subject: [PATCH 0520/1866] Updated JSON foreign key tables test for #1525 --- tests/test_api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index 8b3fcd75..400dae7e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1357,30 +1357,35 @@ def test_row_foreign_key_tables(app_client): "column": "id", "other_column": "foreign_key_with_blank_label", "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1", }, { "other_table": "foreign_key_references", "column": "id", "other_column": "foreign_key_with_label", "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1", }, { "other_table": "complex_foreign_keys", "column": "id", "other_column": "f3", "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1", }, { "other_table": "complex_foreign_keys", "column": "id", "other_column": "f2", "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1", }, { "other_table": "complex_foreign_keys", "column": "id", "other_column": "f1", "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1", }, ] From 7c02be2ee94cc64b120cc58b7a72cd387031f287 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 22:45:37 -0800 Subject: [PATCH 0521/1866] Release 0.59.4 Refs #1525, #1527 --- datasette/version.py | 2 +- docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0ba55573..9c85b763 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.59.3" +__version__ = "0.59.4" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 449ce412..9ddc2794 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_59_4: + +0.59.4 (2021-11-29) +------------------- + +- Fixed bug where columns with a leading underscore could not be removed from the interactive filters list. (:issue:`1527`) +- Fixed bug where columns with a leading underscore were not correctly linked to by the "Links from other tables" interface on the row page. (:issue:`1525`) +- Upgraded dependencies ``aiofiles``, ``black`` and ``janus``. + .. _v0_59_3: 0.59.3 (2021-11-20) From 36b596e3832f6126bb0e4e90cf9257b9e9c9a55e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Dec 2021 11:41:56 -0800 Subject: [PATCH 0522/1866] Framework :: Datasette Trove classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 3cb657e3..9b5bab61 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ setup( tests_require=["datasette[test]"], classifiers=[ "Development Status :: 4 - Beta", + "Framework :: Datasette", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: End Users/Desktop", From 737115ea14cd51ffb55dea886e6a684c148db2c9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Dec 2021 12:03:42 -0800 Subject: [PATCH 0523/1866] Label column finder is now case-insensitive Closes #1544 --- datasette/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index 9f3bbddc..d1217e18 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -286,7 +286,7 @@ class Database: return explicit_label_column column_names = await self.execute_fn(lambda conn: table_columns(conn, table)) # Is there a name or title column? - name_or_title = [c for c in column_names if c in ("name", "title")] + name_or_title = [c for c in column_names if c.lower() in ("name", "title")] if name_or_title: return name_or_title[0] # If a table has two columns, one of which is ID, then label_column is the other one From 1876975e3b120298cec2ff14825260f4a19a0568 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Dec 2021 19:06:45 -0800 Subject: [PATCH 0524/1866] Refactor table view HTML tests to test_table_html.py Refs #1518 --- tests/test_html.py | 1064 +------------------------------------- tests/test_table_html.py | 1045 +++++++++++++++++++++++++++++++++++++ tests/utils.py | 24 + 3 files changed, 1070 insertions(+), 1063 deletions(-) create mode 100644 tests/test_table_html.py create mode 100644 tests/utils.py diff --git a/tests/test_html.py b/tests/test_html.py index a7cb105c..bfe5c8f9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -9,11 +9,11 @@ from .fixtures import ( # noqa make_app_client, METADATA, ) +from .utils import assert_footer_links, inner_html import json import pathlib import pytest import re -import textwrap import urllib.parse @@ -180,67 +180,6 @@ def test_row_strange_table_name_with_url_hash(app_client_with_hash): assert response.status == 200 -@pytest.mark.parametrize( - "path,expected_definition_sql", - [ - ( - "/fixtures/facet_cities", - """ -CREATE TABLE facet_cities ( - id integer primary key, - name text -); - """.strip(), - ), - ( - "/fixtures/compound_three_primary_keys", - """ -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - """.strip(), - ), - ], -) -def test_definition_sql(path, expected_definition_sql, app_client): - response = app_client.get(path) - pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql") - assert expected_definition_sql == pre.string - - -def test_table_cell_truncation(): - with make_app_client(settings={"truncate_cells_html": 5}) as client: - response = client.get("/fixtures/facetable") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - assert table["class"] == ["rows-and-columns"] - assert [ - "Missi…", - "Dogpa…", - "SOMA", - "Tende…", - "Berna…", - "Hayes…", - "Holly…", - "Downt…", - "Los F…", - "Korea…", - "Downt…", - "Greek…", - "Corkt…", - "Mexic…", - "Arcad…", - ] == [ - td.string - for td in table.findAll("td", {"class": "col-neighborhood-b352a7"}) - ] - - def test_row_page_does_not_truncate(): with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable/1") @@ -253,343 +192,6 @@ def test_row_page_does_not_truncate(): ] -def test_add_filter_redirects(app_client): - filter_args = urllib.parse.urlencode( - {"_filter_column": "content", "_filter_op": "startswith", "_filter_value": "x"} - ) - path_base = "/fixtures/simple_primary_key" - path = path_base + "?" + filter_args - response = app_client.get(path) - assert response.status == 302 - assert response.headers["Location"].endswith("?content__startswith=x") - - # Adding a redirect to an existing query string: - path = path_base + "?foo=bar&" + filter_args - response = app_client.get(path) - assert response.status == 302 - assert response.headers["Location"].endswith("?foo=bar&content__startswith=x") - - # Test that op with a __x suffix overrides the filter value - path = ( - path_base - + "?" - + urllib.parse.urlencode( - { - "_filter_column": "content", - "_filter_op": "isnull__5", - "_filter_value": "x", - } - ) - ) - response = app_client.get(path) - assert response.status == 302 - assert response.headers["Location"].endswith("?content__isnull=5") - - -def test_existing_filter_redirects(app_client): - filter_args = { - "_filter_column_1": "name", - "_filter_op_1": "contains", - "_filter_value_1": "hello", - "_filter_column_2": "age", - "_filter_op_2": "gte", - "_filter_value_2": "22", - "_filter_column_3": "age", - "_filter_op_3": "lt", - "_filter_value_3": "30", - "_filter_column_4": "name", - "_filter_op_4": "contains", - "_filter_value_4": "world", - } - path_base = "/fixtures/simple_primary_key" - path = path_base + "?" + urllib.parse.urlencode(filter_args) - response = app_client.get(path) - assert response.status == 302 - assert_querystring_equal( - "name__contains=hello&age__gte=22&age__lt=30&name__contains=world", - response.headers["Location"].split("?")[1], - ) - - # Setting _filter_column_3 to empty string should remove *_3 entirely - filter_args["_filter_column_3"] = "" - path = path_base + "?" + urllib.parse.urlencode(filter_args) - response = app_client.get(path) - assert response.status == 302 - assert_querystring_equal( - "name__contains=hello&age__gte=22&name__contains=world", - response.headers["Location"].split("?")[1], - ) - - # ?_filter_op=exact should be removed if unaccompanied by _fiter_column - response = app_client.get(path_base + "?_filter_op=exact") - assert response.status == 302 - assert "?" not in response.headers["Location"] - - -def test_exact_parameter_results_in_correct_hidden_fields(app_client): - # https://github.com/simonw/datasette/issues/1527 - response = app_client.get( - "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown" - ) - # In this case we should NOT have a hidden _neighborhood__exact=Downtown field - form = Soup(response.body, "html.parser").find("form") - hidden_inputs = { - input["name"]: input["value"] for input in form.select("input[type=hidden]") - } - assert hidden_inputs == {"_facet": "_neighborhood"} - - -def test_empty_search_parameter_gets_removed(app_client): - path_base = "/fixtures/simple_primary_key" - path = ( - path_base - + "?" - + urllib.parse.urlencode( - { - "_search": "", - "_filter_column": "name", - "_filter_op": "exact", - "_filter_value": "chidi", - } - ) - ) - response = app_client.get(path) - assert response.status == 302 - assert response.headers["Location"].endswith("?name__exact=chidi") - - -def test_searchable_view_persists_fts_table(app_client): - # The search form should persist ?_fts_table as a hidden field - response = app_client.get( - "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk" - ) - inputs = Soup(response.body, "html.parser").find("form").findAll("input") - hiddens = [i for i in inputs if i["type"] == "hidden"] - assert [("_fts_table", "searchable_fts"), ("_fts_pk", "pk")] == [ - (hidden["name"], hidden["value"]) for hidden in hiddens - ] - - -def test_sort_by_desc_redirects(app_client): - path_base = "/fixtures/sortable" - path = ( - path_base - + "?" - + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"}) - ) - response = app_client.get(path) - assert response.status == 302 - assert response.headers["Location"].endswith("?_sort_desc=sortable") - - -def test_sort_links(app_client): - response = app_client.get("/fixtures/sortable?_sort=sortable") - assert response.status == 200 - ths = Soup(response.body, "html.parser").findAll("th") - attrs_and_link_attrs = [ - { - "attrs": th.attrs, - "a_href": (th.find("a")["href"] if th.find("a") else None), - } - for th in ths - ] - assert attrs_and_link_attrs == [ - { - "attrs": { - "class": ["col-Link"], - "scope": "col", - "data-column": "Link", - "data-column-type": "", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": None, - }, - { - "attrs": { - "class": ["col-pk1"], - "scope": "col", - "data-column": "pk1", - "data-column-type": "varchar(30)", - "data-column-not-null": "0", - "data-is-pk": "1", - }, - "a_href": None, - }, - { - "attrs": { - "class": ["col-pk2"], - "scope": "col", - "data-column": "pk2", - "data-column-type": "varchar(30)", - "data-column-not-null": "0", - "data-is-pk": "1", - }, - "a_href": None, - }, - { - "attrs": { - "class": ["col-content"], - "scope": "col", - "data-column": "content", - "data-column-type": "text", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": None, - }, - { - "attrs": { - "class": ["col-sortable"], - "scope": "col", - "data-column": "sortable", - "data-column-type": "integer", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": "/fixtures/sortable?_sort_desc=sortable", - }, - { - "attrs": { - "class": ["col-sortable_with_nulls"], - "scope": "col", - "data-column": "sortable_with_nulls", - "data-column-type": "real", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": "/fixtures/sortable?_sort=sortable_with_nulls", - }, - { - "attrs": { - "class": ["col-sortable_with_nulls_2"], - "scope": "col", - "data-column": "sortable_with_nulls_2", - "data-column-type": "real", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": "/fixtures/sortable?_sort=sortable_with_nulls_2", - }, - { - "attrs": { - "class": ["col-text"], - "scope": "col", - "data-column": "text", - "data-column-type": "text", - "data-column-not-null": "0", - "data-is-pk": "0", - }, - "a_href": "/fixtures/sortable?_sort=text", - }, - ] - - -def test_facet_display(app_client): - response = app_client.get( - "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth" - ) - assert response.status == 200 - soup = Soup(response.body, "html.parser") - divs = soup.find("div", {"class": "facet-results"}).findAll("div") - actual = [] - for div in divs: - actual.append( - { - "name": div.find("strong").text.split()[0], - "items": [ - { - "name": a.text, - "qs": a["href"].split("?")[-1], - "count": int(str(a.parent).split("")[1].split("<")[0]), - } - for a in div.find("ul").findAll("a") - ], - } - ) - assert actual == [ - { - "name": "_city_id", - "items": [ - { - "name": "San Francisco", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1", - "count": 6, - }, - { - "name": "Los Angeles", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2", - "count": 4, - }, - { - "name": "Detroit", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3", - "count": 4, - }, - { - "name": "Memnonia", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4", - "count": 1, - }, - ], - }, - { - "name": "planet_int", - "items": [ - { - "name": "1", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1", - "count": 14, - }, - { - "name": "2", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2", - "count": 1, - }, - ], - }, - { - "name": "on_earth", - "items": [ - { - "name": "1", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1", - "count": 14, - }, - { - "name": "0", - "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0", - "count": 1, - }, - ], - }, - ] - - -def test_facets_persist_through_filter_form(app_client): - response = app_client.get( - "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags" - ) - assert response.status == 200 - inputs = Soup(response.body, "html.parser").find("form").findAll("input") - hiddens = [i for i in inputs if i["type"] == "hidden"] - assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [ - ("_facet", "planet_int"), - ("_facet", "_city_id"), - ("_facet_array", "tags"), - ] - - -def test_next_does_not_persist_in_hidden_field(app_client): - response = app_client.get("/fixtures/searchable?_size=1&_next=1") - assert response.status == 200 - inputs = Soup(response.body, "html.parser").find("form").findAll("input") - hiddens = [i for i in inputs if i["type"] == "hidden"] - assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [ - ("_size", "1"), - ] - - @pytest.mark.parametrize( "path,expected_classes", [ @@ -646,74 +248,6 @@ def test_templates_considered(app_client, path, expected_considered): assert f"" in response.text -def test_table_html_simple_primary_key(app_client): - response = app_client.get("/fixtures/simple_primary_key?_size=3") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - assert table["class"] == ["rows-and-columns"] - ths = table.findAll("th") - assert "id\xa0▼" == ths[0].find("a").string.strip() - for expected_col, th in zip(("content",), ths[1:]): - a = th.find("a") - assert expected_col == a.string - assert a["href"].endswith(f"/simple_primary_key?_size=3&_sort={expected_col}") - assert ["nofollow"] == a["rel"] - assert [ - [ - '
', - '', - ], - [ - '', - '', - ], - [ - '', - '', - ], - ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - - -def test_table_csv_json_export_interface(app_client): - response = app_client.get("/fixtures/simple_primary_key?id__gt=2") - assert response.status == 200 - # The links at the top of the page - links = ( - Soup(response.body, "html.parser") - .find("p", {"class": "export-links"}) - .findAll("a") - ) - actual = [l["href"] for l in links] - expected = [ - "/fixtures/simple_primary_key.json?id__gt=2", - "/fixtures/simple_primary_key.testall?id__gt=2", - "/fixtures/simple_primary_key.testnone?id__gt=2", - "/fixtures/simple_primary_key.testresponse?id__gt=2", - "/fixtures/simple_primary_key.csv?id__gt=2&_size=max", - "#export", - ] - assert expected == actual - # And the advaced export box at the bottom: - div = Soup(response.body, "html.parser").find("div", {"class": "advanced-export"}) - json_links = [a["href"] for a in div.find("p").findAll("a")] - assert [ - "/fixtures/simple_primary_key.json?id__gt=2", - "/fixtures/simple_primary_key.json?id__gt=2&_shape=array", - "/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on", - "/fixtures/simple_primary_key.json?id__gt=2&_shape=object", - ] == json_links - # And the CSV form - form = div.find("form") - assert form["action"].endswith("/simple_primary_key.csv") - inputs = [str(input) for input in form.findAll("input")] - assert [ - '', - '', - '', - '', - ] == inputs - - def test_row_json_export_link(app_client): response = app_client.get("/fixtures/simple_primary_key/1") assert response.status == 200 @@ -727,26 +261,6 @@ def test_query_json_csv_export_links(app_client): assert 'CSV' in response.text -def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): - response = app_client.get("/fixtures/facetable") - assert response.status == 200 - links = ( - Soup(response.body, "html.parser") - .find("p", {"class": "export-links"}) - .findAll("a") - ) - actual = [l["href"] for l in links] - expected = [ - "/fixtures/facetable.json?_labels=on", - "/fixtures/facetable.testall?_labels=on", - "/fixtures/facetable.testnone?_labels=on", - "/fixtures/facetable.testresponse?_labels=on", - "/fixtures/facetable.csv?_labels=on&_size=max", - "#export", - ] - assert expected == actual - - def test_row_html_simple_primary_key(app_client): response = app_client.get("/fixtures/simple_primary_key/1") assert response.status == 200 @@ -760,45 +274,6 @@ def test_row_html_simple_primary_key(app_client): ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] -def test_table_not_exists(app_client): - assert "Table not found: blah" in app_client.get("/fixtures/blah").text - - -def test_table_html_no_primary_key(app_client): - response = app_client.get("/fixtures/no_primary_key") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - # We have disabled sorting for this table using metadata.json - assert ["content", "a", "b", "c"] == [ - th.string.strip() for th in table.select("thead th")[2:] - ] - expected = [ - [ - ''.format( - i, i - ), - f'', - f'', - f'', - f'', - f'', - ] - for i in range(1, 51) - ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - -def test_rowid_sortable_no_primary_key(app_client): - response = app_client.get("/fixtures/no_primary_key") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - assert table["class"] == ["rows-and-columns"] - ths = table.findAll("th") - assert "rowid\xa0▼" == ths[1].find("a").string.strip() - - def test_row_html_no_primary_key(app_client): response = app_client.get("/fixtures/no_primary_key/1") assert response.status == 200 @@ -848,143 +323,6 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l assert link == expected_link -def test_table_html_compound_primary_key(app_client): - response = app_client.get("/fixtures/compound_primary_key") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - ths = table.findAll("th") - assert "Link" == ths[0].string.strip() - for expected_col, th in zip(("pk1", "pk2", "content"), ths[1:]): - a = th.find("a") - assert expected_col == a.string - assert th["class"] == [f"col-{expected_col}"] - assert a["href"].endswith(f"/compound_primary_key?_sort={expected_col}") - expected = [ - [ - '', - '', - '', - '', - ] - ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - -def test_table_html_foreign_key_links(app_client): - response = app_client.get("/fixtures/foreign_key_references") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - assert actual == [ - [ - '', - '', - '', - '', - '', - '', - ], - [ - '', - '', - '', - '', - '', - '', - ], - ] - - -def test_table_html_foreign_key_facets(app_client): - response = app_client.get( - "/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label" - ) - assert response.status == 200 - assert ( - '
  • ' - "- 1
    • " - ) in response.text - - -def test_table_html_disable_foreign_key_links_with_labels(app_client): - response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - assert actual == [ - [ - '', - '', - '', - '', - '', - '', - ] - ] - - -def test_table_html_foreign_key_custom_label_column(app_client): - response = app_client.get("/fixtures/custom_foreign_key_label") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - expected = [ - [ - '', - '', - ] - ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - -@pytest.mark.parametrize( - "path,expected_column_options", - [ - ("/fixtures/infinity", ["- column -", "rowid", "value"]), - ( - "/fixtures/primary_key_multiple_columns", - ["- column -", "id", "content", "content2"], - ), - ("/fixtures/compound_primary_key", ["- column -", "pk1", "pk2", "content"]), - ], -) -def test_table_html_filter_form_column_options( - path, expected_column_options, app_client -): - response = app_client.get(path) - assert response.status == 200 - form = Soup(response.body, "html.parser").find("form") - column_options = [ - o.attrs.get("value") or o.string - for o in form.select("select[name=_filter_column] option") - ] - assert expected_column_options == column_options - - -def test_table_html_filter_form_still_shows_nocol_columns(app_client): - # https://github.com/simonw/datasette/issues/1503 - response = app_client.get("/fixtures/sortable?_nocol=sortable") - assert response.status == 200 - form = Soup(response.body, "html.parser").find("form") - assert [ - o.string - for o in form.select("select[name='_filter_column']")[0].select("option") - ] == [ - "- column -", - "pk1", - "pk2", - "content", - "sortable_with_nulls", - "sortable_with_nulls_2", - "text", - # Moved to the end because it is no longer returned by the query: - "sortable", - ] - - def test_row_html_compound_primary_key(app_client): response = app_client.get("/fixtures/compound_primary_key/a,b") assert response.status == 200 @@ -1004,58 +342,6 @@ def test_row_html_compound_primary_key(app_client): ] -def test_compound_primary_key_with_foreign_key_references(app_client): - # e.g. a many-to-many table with a compound primary key on the two columns - response = app_client.get("/fixtures/searchable_tags") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - expected = [ - [ - '', - '', - '', - ], - [ - '', - '', - '', - ], - ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - -def test_view_html(app_client): - response = app_client.get("/fixtures/simple_view?_size=3") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - ths = table.select("thead th") - assert 2 == len(ths) - assert ths[0].find("a") is not None - assert ths[0].find("a")["href"].endswith("/simple_view?_size=3&_sort=content") - assert ths[0].find("a").string.strip() == "content" - assert ths[1].find("a") is None - assert ths[1].string.strip() == "upper_content" - expected = [ - [ - '', - '', - ], - [ - '', - '', - ], - [ - '', - '', - ], - ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - def test_index_metadata(app_client): response = app_client.get("/") assert response.status == 200 @@ -1094,20 +380,6 @@ def test_database_metadata_with_custom_sql(app_client): assert_footer_links(soup) -def test_table_metadata(app_client): - response = app_client.get("/fixtures/simple_primary_key") - assert response.status == 200 - soup = Soup(response.body, "html.parser") - # Page title should be custom and should be HTML escaped - assert "This <em>HTML</em> is escaped" == inner_html(soup.find("h1")) - # Description should be custom and NOT escaped (we used description_html) - assert "Simple primary key" == inner_html( - soup.find("div", {"class": "metadata-description"}) - ) - # The source/license should be inherited - assert_footer_links(soup) - - def test_database_download_for_immutable(): with make_app_client(is_immutable=True) as client: assert not client.ds.databases["fixtures"].is_mutable @@ -1169,36 +441,6 @@ def test_allow_sql_off(): assert b"View and edit SQL" not in response.body -def assert_querystring_equal(expected, actual): - assert sorted(expected.split("&")) == sorted(actual.split("&")) - - -def assert_footer_links(soup): - footer_links = soup.find("footer").findAll("a") - assert 4 == len(footer_links) - datasette_link, license_link, source_link, about_link = footer_links - assert "Datasette" == datasette_link.text.strip() - assert "tests/fixtures.py" == source_link.text.strip() - assert "Apache License 2.0" == license_link.text.strip() - assert "About Datasette" == about_link.text.strip() - assert "https://datasette.io/" == datasette_link["href"] - assert ( - "https://github.com/simonw/datasette/blob/main/tests/fixtures.py" - == source_link["href"] - ) - assert ( - "https://github.com/simonw/datasette/blob/main/LICENSE" == license_link["href"] - ) - assert "https://github.com/simonw/datasette" == about_link["href"] - - -def inner_html(soup): - html = str(soup) - # This includes the parent tag - so remove that - inner_html = html.split(">", 1)[1].rsplit("<", 1)[0] - return inner_html.strip() - - @pytest.mark.parametrize("path", ["/404", "/fixtures/404"]) def test_404(app_client, path): response = app_client.get(path) @@ -1249,31 +491,6 @@ def test_canned_query_with_custom_metadata(app_client): ) -@pytest.mark.parametrize( - "path,has_object,has_stream,has_expand", - [ - ("/fixtures/no_primary_key", False, True, False), - ("/fixtures/complex_foreign_keys", True, False, True), - ], -) -def test_advanced_export_box(app_client, path, has_object, has_stream, has_expand): - response = app_client.get(path) - assert response.status == 200 - soup = Soup(response.body, "html.parser") - # JSON shape options - expected_json_shapes = ["default", "array", "newline-delimited"] - if has_object: - expected_json_shapes.append("object") - div = soup.find("div", {"class": "advanced-export"}) - assert expected_json_shapes == [a.text for a in div.find("p").findAll("a")] - # "stream all rows" option - if has_stream: - assert "stream all rows" in str(div) - # "expand labels" option - if has_expand: - assert "expand labels" in str(div) - - def test_urlify_custom_queries(app_client): path = "/fixtures?" + urllib.parse.urlencode( {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"} @@ -1376,91 +593,6 @@ def test_canned_query_show_hide_metadata_option( assert '1', - '', - '', - ], - [ - '', - '', - '', - ], - [ - '', - '', - '', - ], - ] - assert expected_tds == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] - - def test_binary_data_display_in_query(app_client): response = app_client.get("/fixtures?sql=select+*+from+binary_data") assert response.status == 200 @@ -1525,19 +657,6 @@ def test_metadata_json_html(app_client): assert METADATA == json.loads(pre.text) -def test_custom_table_include(): - with make_app_client( - template_dir=str(pathlib.Path(__file__).parent / "test_templates") - ) as client: - response = client.get("/fixtures/complex_foreign_keys") - assert response.status == 200 - assert ( - '
    ' - '1 - 2 - hello 1' - "
    " - ) == str(Soup(response.text, "html.parser").select_one("div.custom-table-row")) - - @pytest.mark.parametrize( "path", [ @@ -1584,68 +703,6 @@ def test_debug_context_includes_extra_template_vars(): assert "scope_path" in response.text -def test_metadata_sort(app_client): - response = app_client.get("/fixtures/facet_cities") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - assert table["class"] == ["rows-and-columns"] - ths = table.findAll("th") - assert ["id", "name\xa0▼"] == [th.find("a").string.strip() for th in ths] - rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - expected = [ - [ - '', - '', - ], - [ - '', - '', - ], - [ - '', - '', - ], - [ - '', - '', - ], - ] - assert expected == rows - # Make sure you can reverse that sort order - response = app_client.get("/fixtures/facet_cities?_sort_desc=name") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - assert list(reversed(expected)) == rows - - -def test_metadata_sort_desc(app_client): - response = app_client.get("/fixtures/attraction_characteristic") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - assert table["class"] == ["rows-and-columns"] - ths = table.findAll("th") - assert ["pk\xa0▲", "name"] == [th.find("a").string.strip() for th in ths] - rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - expected = [ - [ - '', - '', - ], - [ - '', - '', - ], - ] - assert expected == rows - # Make sure you can reverse that sort order - response = app_client.get("/fixtures/attraction_characteristic?_sort=pk") - assert response.status == 200 - table = Soup(response.body, "html.parser").find("table") - rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] - assert list(reversed(expected)) == rows - - @pytest.mark.parametrize( "path", [ @@ -1787,126 +844,7 @@ def test_navigation_menu_links( ), f"{link} found but should not have been in nav menu" -@pytest.mark.parametrize( - "max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url", - ( - ( - 5, - # Default should show 2 facets - "/fixtures/facetable?_facet=_neighborhood", - 2, - True, - "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", - ), - # _facet_size above max_returned_rows should show max_returned_rows (5) - ( - 5, - "/fixtures/facetable?_facet=_neighborhood&_facet_size=50", - 5, - True, - "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", - ), - # If max_returned_rows is high enough, should return all - ( - 20, - "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", - 14, - False, - None, - ), - # If num facets > max_returned_rows, show ... without a link - # _facet_size above max_returned_rows should show max_returned_rows (5) - ( - 5, - "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", - 5, - True, - None, - ), - ), -) -def test_facet_more_links( - max_returned_rows, - path, - expected_num_facets, - expected_ellipses, - expected_ellipses_url, -): - with make_app_client( - settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2} - ) as client: - response = client.get(path) - soup = Soup(response.body, "html.parser") - lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)") - facet_truncated = soup.select_one(".facet-truncated") - assert len(lis) == expected_num_facets - if not expected_ellipses: - assert facet_truncated is None - else: - if expected_ellipses_url: - assert facet_truncated.find("a")["href"] == expected_ellipses_url - else: - assert facet_truncated.find("a") is None - - -def test_unavailable_table_does_not_break_sort_relationships(): - # https://github.com/simonw/datasette/issues/1305 - with make_app_client( - metadata={ - "databases": { - "fixtures": {"tables": {"foreign_key_references": {"allow": False}}} - } - } - ) as client: - response = client.get("/?_sort=relationships") - assert response.status == 200 - - def test_trace_correctly_escaped(app_client): response = app_client.get("/fixtures?sql=select+'

    Hello'&_trace=1") assert "select '

    Hello" not in response.text assert "select '<h1>Hello" in response.text - - -def test_column_metadata(app_client): - response = app_client.get("/fixtures/roadside_attractions") - soup = Soup(response.body, "html.parser") - dl = soup.find("dl") - assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [ - ("name", "The name of the attraction"), - ("address", "The street address for the attraction"), - ] - assert ( - soup.select("th[data-column=name]")[0]["data-column-description"] - == "The name of the attraction" - ) - assert ( - soup.select("th[data-column=address]")[0]["data-column-description"] - == "The street address for the attraction" - ) - - -@pytest.mark.parametrize("use_facet_size_max", (True, False)) -def test_facet_total_shown_if_facet_max_size(use_facet_size_max): - # https://github.com/simonw/datasette/issues/1423 - with make_app_client(settings={"max_returned_rows": 100}) as client: - path = "/fixtures/sortable?_facet=content&_facet=pk1" - if use_facet_size_max: - path += "&_facet_size=max" - response = client.get(path) - assert response.status == 200 - fragments = ( - '>100', - '8', - ) - for fragment in fragments: - if use_facet_size_max: - assert fragment in response.text - else: - assert fragment not in response.text - - -def test_sort_rowid_with_next(app_client): - # https://github.com/simonw/datasette/issues/1470 - response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid") - assert response.status == 200 diff --git a/tests/test_table_html.py b/tests/test_table_html.py new file mode 100644 index 00000000..2fbb53bd --- /dev/null +++ b/tests/test_table_html.py @@ -0,0 +1,1045 @@ +from bs4 import BeautifulSoup as Soup +from .fixtures import ( # noqa + app_client, + make_app_client, +) +import pathlib +import pytest +import urllib.parse +from .utils import assert_footer_links, inner_html + + +@pytest.mark.parametrize( + "path,expected_definition_sql", + [ + ( + "/fixtures/facet_cities", + """ +CREATE TABLE facet_cities ( + id integer primary key, + name text +); + """.strip(), + ), + ( + "/fixtures/compound_three_primary_keys", + """ +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + """.strip(), + ), + ], +) +def test_table_definition_sql(path, expected_definition_sql, app_client): + response = app_client.get(path) + pre = Soup(response.body, "html.parser").select_one("pre.wrapped-sql") + assert expected_definition_sql == pre.string + + +def test_table_cell_truncation(): + with make_app_client(settings={"truncate_cells_html": 5}) as client: + response = client.get("/fixtures/facetable") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + assert table["class"] == ["rows-and-columns"] + assert [ + "Missi…", + "Dogpa…", + "SOMA", + "Tende…", + "Berna…", + "Hayes…", + "Holly…", + "Downt…", + "Los F…", + "Korea…", + "Downt…", + "Greek…", + "Corkt…", + "Mexic…", + "Arcad…", + ] == [ + td.string + for td in table.findAll("td", {"class": "col-neighborhood-b352a7"}) + ] + + +def test_add_filter_redirects(app_client): + filter_args = urllib.parse.urlencode( + {"_filter_column": "content", "_filter_op": "startswith", "_filter_value": "x"} + ) + path_base = "/fixtures/simple_primary_key" + path = path_base + "?" + filter_args + response = app_client.get(path) + assert response.status == 302 + assert response.headers["Location"].endswith("?content__startswith=x") + + # Adding a redirect to an existing query string: + path = path_base + "?foo=bar&" + filter_args + response = app_client.get(path) + assert response.status == 302 + assert response.headers["Location"].endswith("?foo=bar&content__startswith=x") + + # Test that op with a __x suffix overrides the filter value + path = ( + path_base + + "?" + + urllib.parse.urlencode( + { + "_filter_column": "content", + "_filter_op": "isnull__5", + "_filter_value": "x", + } + ) + ) + response = app_client.get(path) + assert response.status == 302 + assert response.headers["Location"].endswith("?content__isnull=5") + + +def test_existing_filter_redirects(app_client): + filter_args = { + "_filter_column_1": "name", + "_filter_op_1": "contains", + "_filter_value_1": "hello", + "_filter_column_2": "age", + "_filter_op_2": "gte", + "_filter_value_2": "22", + "_filter_column_3": "age", + "_filter_op_3": "lt", + "_filter_value_3": "30", + "_filter_column_4": "name", + "_filter_op_4": "contains", + "_filter_value_4": "world", + } + path_base = "/fixtures/simple_primary_key" + path = path_base + "?" + urllib.parse.urlencode(filter_args) + response = app_client.get(path) + assert response.status == 302 + assert_querystring_equal( + "name__contains=hello&age__gte=22&age__lt=30&name__contains=world", + response.headers["Location"].split("?")[1], + ) + + # Setting _filter_column_3 to empty string should remove *_3 entirely + filter_args["_filter_column_3"] = "" + path = path_base + "?" + urllib.parse.urlencode(filter_args) + response = app_client.get(path) + assert response.status == 302 + assert_querystring_equal( + "name__contains=hello&age__gte=22&name__contains=world", + response.headers["Location"].split("?")[1], + ) + + # ?_filter_op=exact should be removed if unaccompanied by _fiter_column + response = app_client.get(path_base + "?_filter_op=exact") + assert response.status == 302 + assert "?" not in response.headers["Location"] + + +def test_exact_parameter_results_in_correct_hidden_fields(app_client): + # https://github.com/simonw/datasette/issues/1527 + response = app_client.get( + "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown" + ) + # In this case we should NOT have a hidden _neighborhood__exact=Downtown field + form = Soup(response.body, "html.parser").find("form") + hidden_inputs = { + input["name"]: input["value"] for input in form.select("input[type=hidden]") + } + assert hidden_inputs == {"_facet": "_neighborhood"} + + +def test_empty_search_parameter_gets_removed(app_client): + path_base = "/fixtures/simple_primary_key" + path = ( + path_base + + "?" + + urllib.parse.urlencode( + { + "_search": "", + "_filter_column": "name", + "_filter_op": "exact", + "_filter_value": "chidi", + } + ) + ) + response = app_client.get(path) + assert response.status == 302 + assert response.headers["Location"].endswith("?name__exact=chidi") + + +def test_searchable_view_persists_fts_table(app_client): + # The search form should persist ?_fts_table as a hidden field + response = app_client.get( + "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk" + ) + inputs = Soup(response.body, "html.parser").find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [("_fts_table", "searchable_fts"), ("_fts_pk", "pk")] == [ + (hidden["name"], hidden["value"]) for hidden in hiddens + ] + + +def test_sort_by_desc_redirects(app_client): + path_base = "/fixtures/sortable" + path = ( + path_base + + "?" + + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"}) + ) + response = app_client.get(path) + assert response.status == 302 + assert response.headers["Location"].endswith("?_sort_desc=sortable") + + +def test_sort_links(app_client): + response = app_client.get("/fixtures/sortable?_sort=sortable") + assert response.status == 200 + ths = Soup(response.body, "html.parser").findAll("th") + attrs_and_link_attrs = [ + { + "attrs": th.attrs, + "a_href": (th.find("a")["href"] if th.find("a") else None), + } + for th in ths + ] + assert attrs_and_link_attrs == [ + { + "attrs": { + "class": ["col-Link"], + "scope": "col", + "data-column": "Link", + "data-column-type": "", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": None, + }, + { + "attrs": { + "class": ["col-pk1"], + "scope": "col", + "data-column": "pk1", + "data-column-type": "varchar(30)", + "data-column-not-null": "0", + "data-is-pk": "1", + }, + "a_href": None, + }, + { + "attrs": { + "class": ["col-pk2"], + "scope": "col", + "data-column": "pk2", + "data-column-type": "varchar(30)", + "data-column-not-null": "0", + "data-is-pk": "1", + }, + "a_href": None, + }, + { + "attrs": { + "class": ["col-content"], + "scope": "col", + "data-column": "content", + "data-column-type": "text", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": None, + }, + { + "attrs": { + "class": ["col-sortable"], + "scope": "col", + "data-column": "sortable", + "data-column-type": "integer", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": "/fixtures/sortable?_sort_desc=sortable", + }, + { + "attrs": { + "class": ["col-sortable_with_nulls"], + "scope": "col", + "data-column": "sortable_with_nulls", + "data-column-type": "real", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": "/fixtures/sortable?_sort=sortable_with_nulls", + }, + { + "attrs": { + "class": ["col-sortable_with_nulls_2"], + "scope": "col", + "data-column": "sortable_with_nulls_2", + "data-column-type": "real", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": "/fixtures/sortable?_sort=sortable_with_nulls_2", + }, + { + "attrs": { + "class": ["col-text"], + "scope": "col", + "data-column": "text", + "data-column-type": "text", + "data-column-not-null": "0", + "data-is-pk": "0", + }, + "a_href": "/fixtures/sortable?_sort=text", + }, + ] + + +def test_facet_display(app_client): + response = app_client.get( + "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth" + ) + assert response.status == 200 + soup = Soup(response.body, "html.parser") + divs = soup.find("div", {"class": "facet-results"}).findAll("div") + actual = [] + for div in divs: + actual.append( + { + "name": div.find("strong").text.split()[0], + "items": [ + { + "name": a.text, + "qs": a["href"].split("?")[-1], + "count": int(str(a.parent).split("")[1].split("<")[0]), + } + for a in div.find("ul").findAll("a") + ], + } + ) + assert actual == [ + { + "name": "_city_id", + "items": [ + { + "name": "San Francisco", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1", + "count": 6, + }, + { + "name": "Los Angeles", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2", + "count": 4, + }, + { + "name": "Detroit", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3", + "count": 4, + }, + { + "name": "Memnonia", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4", + "count": 1, + }, + ], + }, + { + "name": "planet_int", + "items": [ + { + "name": "1", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1", + "count": 14, + }, + { + "name": "2", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2", + "count": 1, + }, + ], + }, + { + "name": "on_earth", + "items": [ + { + "name": "1", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1", + "count": 14, + }, + { + "name": "0", + "qs": "_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0", + "count": 1, + }, + ], + }, + ] + + +def test_facets_persist_through_filter_form(app_client): + response = app_client.get( + "/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags" + ) + assert response.status == 200 + inputs = Soup(response.body, "html.parser").find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [ + ("_facet", "planet_int"), + ("_facet", "_city_id"), + ("_facet_array", "tags"), + ] + + +def test_next_does_not_persist_in_hidden_field(app_client): + response = app_client.get("/fixtures/searchable?_size=1&_next=1") + assert response.status == 200 + inputs = Soup(response.body, "html.parser").find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [ + ("_size", "1"), + ] + + +def test_table_html_simple_primary_key(app_client): + response = app_client.get("/fixtures/simple_primary_key?_size=3") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + assert table["class"] == ["rows-and-columns"] + ths = table.findAll("th") + assert "id\xa0▼" == ths[0].find("a").string.strip() + for expected_col, th in zip(("content",), ths[1:]): + a = th.find("a") + assert expected_col == a.string + assert a["href"].endswith(f"/simple_primary_key?_size=3&_sort={expected_col}") + assert ["nofollow"] == a["rel"] + assert [ + [ + '

    ', + '', + ], + [ + '', + '', + ], + [ + '', + '', + ], + ] == [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + + +def test_table_csv_json_export_interface(app_client): + response = app_client.get("/fixtures/simple_primary_key?id__gt=2") + assert response.status == 200 + # The links at the top of the page + links = ( + Soup(response.body, "html.parser") + .find("p", {"class": "export-links"}) + .findAll("a") + ) + actual = [l["href"] for l in links] + expected = [ + "/fixtures/simple_primary_key.json?id__gt=2", + "/fixtures/simple_primary_key.testall?id__gt=2", + "/fixtures/simple_primary_key.testnone?id__gt=2", + "/fixtures/simple_primary_key.testresponse?id__gt=2", + "/fixtures/simple_primary_key.csv?id__gt=2&_size=max", + "#export", + ] + assert expected == actual + # And the advaced export box at the bottom: + div = Soup(response.body, "html.parser").find("div", {"class": "advanced-export"}) + json_links = [a["href"] for a in div.find("p").findAll("a")] + assert [ + "/fixtures/simple_primary_key.json?id__gt=2", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=array", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=object", + ] == json_links + # And the CSV form + form = div.find("form") + assert form["action"].endswith("/simple_primary_key.csv") + inputs = [str(input) for input in form.findAll("input")] + assert [ + '', + '', + '', + '', + ] == inputs + + +def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): + response = app_client.get("/fixtures/facetable") + assert response.status == 200 + links = ( + Soup(response.body, "html.parser") + .find("p", {"class": "export-links"}) + .findAll("a") + ) + actual = [l["href"] for l in links] + expected = [ + "/fixtures/facetable.json?_labels=on", + "/fixtures/facetable.testall?_labels=on", + "/fixtures/facetable.testnone?_labels=on", + "/fixtures/facetable.testresponse?_labels=on", + "/fixtures/facetable.csv?_labels=on&_size=max", + "#export", + ] + assert expected == actual + + +def test_table_not_exists(app_client): + assert "Table not found: blah" in app_client.get("/fixtures/blah").text + + +def test_table_html_no_primary_key(app_client): + response = app_client.get("/fixtures/no_primary_key") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + # We have disabled sorting for this table using metadata.json + assert ["content", "a", "b", "c"] == [ + th.string.strip() for th in table.select("thead th")[2:] + ] + expected = [ + [ + ''.format( + i, i + ), + f'', + f'', + f'', + f'', + f'', + ] + for i in range(1, 51) + ] + assert expected == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +def test_rowid_sortable_no_primary_key(app_client): + response = app_client.get("/fixtures/no_primary_key") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + assert table["class"] == ["rows-and-columns"] + ths = table.findAll("th") + assert "rowid\xa0▼" == ths[1].find("a").string.strip() + + +def test_table_html_compound_primary_key(app_client): + response = app_client.get("/fixtures/compound_primary_key") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + ths = table.findAll("th") + assert "Link" == ths[0].string.strip() + for expected_col, th in zip(("pk1", "pk2", "content"), ths[1:]): + a = th.find("a") + assert expected_col == a.string + assert th["class"] == [f"col-{expected_col}"] + assert a["href"].endswith(f"/compound_primary_key?_sort={expected_col}") + expected = [ + [ + '', + '', + '', + '', + ] + ] + assert expected == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +def test_table_html_foreign_key_links(app_client): + response = app_client.get("/fixtures/foreign_key_references") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert actual == [ + [ + '', + '', + '', + '', + '', + '', + ], + [ + '', + '', + '', + '', + '', + '', + ], + ] + + +def test_table_html_foreign_key_facets(app_client): + response = app_client.get( + "/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label" + ) + assert response.status == 200 + assert ( + '
  • ' + "- 1
    • " + ) in response.text + + +def test_table_html_disable_foreign_key_links_with_labels(app_client): + response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert actual == [ + [ + '', + '', + '', + '', + '', + '', + ] + ] + + +def test_table_html_foreign_key_custom_label_column(app_client): + response = app_client.get("/fixtures/custom_foreign_key_label") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + expected = [ + [ + '', + '', + ] + ] + assert expected == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +@pytest.mark.parametrize( + "path,expected_column_options", + [ + ("/fixtures/infinity", ["- column -", "rowid", "value"]), + ( + "/fixtures/primary_key_multiple_columns", + ["- column -", "id", "content", "content2"], + ), + ("/fixtures/compound_primary_key", ["- column -", "pk1", "pk2", "content"]), + ], +) +def test_table_html_filter_form_column_options( + path, expected_column_options, app_client +): + response = app_client.get(path) + assert response.status == 200 + form = Soup(response.body, "html.parser").find("form") + column_options = [ + o.attrs.get("value") or o.string + for o in form.select("select[name=_filter_column] option") + ] + assert expected_column_options == column_options + + +def test_table_html_filter_form_still_shows_nocol_columns(app_client): + # https://github.com/simonw/datasette/issues/1503 + response = app_client.get("/fixtures/sortable?_nocol=sortable") + assert response.status == 200 + form = Soup(response.body, "html.parser").find("form") + assert [ + o.string + for o in form.select("select[name='_filter_column']")[0].select("option") + ] == [ + "- column -", + "pk1", + "pk2", + "content", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + # Moved to the end because it is no longer returned by the query: + "sortable", + ] + + +def test_compound_primary_key_with_foreign_key_references(app_client): + # e.g. a many-to-many table with a compound primary key on the two columns + response = app_client.get("/fixtures/searchable_tags") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + expected = [ + [ + '', + '', + '', + ], + [ + '', + '', + '', + ], + ] + assert expected == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +def test_view_html(app_client): + response = app_client.get("/fixtures/simple_view?_size=3") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + ths = table.select("thead th") + assert 2 == len(ths) + assert ths[0].find("a") is not None + assert ths[0].find("a")["href"].endswith("/simple_view?_size=3&_sort=content") + assert ths[0].find("a").string.strip() == "content" + assert ths[1].find("a") is None + assert ths[1].string.strip() == "upper_content" + expected = [ + [ + '', + '', + ], + [ + '', + '', + ], + [ + '', + '', + ], + ] + assert expected == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +def test_table_metadata(app_client): + response = app_client.get("/fixtures/simple_primary_key") + assert response.status == 200 + soup = Soup(response.body, "html.parser") + # Page title should be custom and should be HTML escaped + assert "This <em>HTML</em> is escaped" == inner_html(soup.find("h1")) + # Description should be custom and NOT escaped (we used description_html) + assert "Simple primary key" == inner_html( + soup.find("div", {"class": "metadata-description"}) + ) + # The source/license should be inherited + assert_footer_links(soup) + + +@pytest.mark.parametrize( + "path,has_object,has_stream,has_expand", + [ + ("/fixtures/no_primary_key", False, True, False), + ("/fixtures/complex_foreign_keys", True, False, True), + ], +) +def test_advanced_export_box(app_client, path, has_object, has_stream, has_expand): + response = app_client.get(path) + assert response.status == 200 + soup = Soup(response.body, "html.parser") + # JSON shape options + expected_json_shapes = ["default", "array", "newline-delimited"] + if has_object: + expected_json_shapes.append("object") + div = soup.find("div", {"class": "advanced-export"}) + assert expected_json_shapes == [a.text for a in div.find("p").findAll("a")] + # "stream all rows" option + if has_stream: + assert "stream all rows" in str(div) + # "expand labels" option + if has_expand: + assert "expand labels" in str(div) + + +def test_extra_where_clauses(app_client): + response = app_client.get( + "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=_city_id=1" + ) + soup = Soup(response.body, "html.parser") + div = soup.select(".extra-wheres")[0] + assert "2 extra where clauses" == div.find("h3").text + hrefs = [a["href"] for a in div.findAll("a")] + assert [ + "/fixtures/facetable?_where=_city_id%3D1", + "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27", + ] == hrefs + # These should also be persisted as hidden fields + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "_city_id=1")] == [ + (hidden["name"], hidden["value"]) for hidden in hiddens + ] + + +@pytest.mark.parametrize( + "path,expected_hidden", + [ + ("/fixtures/facetable?_size=10", [("_size", "10")]), + ( + "/fixtures/facetable?_size=10&_ignore=1&_ignore=2", + [ + ("_size", "10"), + ("_ignore", "1"), + ("_ignore", "2"), + ], + ), + ], +) +def test_other_hidden_form_fields(app_client, path, expected_hidden): + response = app_client.get(path) + soup = Soup(response.body, "html.parser") + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden + + +@pytest.mark.parametrize( + "path,expected_hidden", + [ + ("/fixtures/searchable?_search=terry", []), + ("/fixtures/searchable?_sort=text2", []), + ("/fixtures/searchable?_sort=text2&_where=1", [("_where", "1")]), + ], +) +def test_search_and_sort_fields_not_duplicated(app_client, path, expected_hidden): + # https://github.com/simonw/datasette/issues/1214 + response = app_client.get(path) + soup = Soup(response.body, "html.parser") + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden + + +def test_binary_data_display_in_table(app_client): + response = app_client.get("/fixtures/binary_data") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + expected_tds = [ + [ + '', + '', + '', + ], + [ + '', + '', + '', + ], + [ + '', + '', + '', + ], + ] + assert expected_tds == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + +def test_custom_table_include(): + with make_app_client( + template_dir=str(pathlib.Path(__file__).parent / "test_templates") + ) as client: + response = client.get("/fixtures/complex_foreign_keys") + assert response.status == 200 + assert ( + '
    ' + '1 - 2 - hello 1' + "
    " + ) == str(Soup(response.text, "html.parser").select_one("div.custom-table-row")) + + +def test_metadata_sort(app_client): + response = app_client.get("/fixtures/facet_cities") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + assert table["class"] == ["rows-and-columns"] + ths = table.findAll("th") + assert ["id", "name\xa0▼"] == [th.find("a").string.strip() for th in ths] + rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + expected = [ + [ + '', + '', + ], + [ + '', + '', + ], + [ + '', + '', + ], + [ + '', + '', + ], + ] + assert expected == rows + # Make sure you can reverse that sort order + response = app_client.get("/fixtures/facet_cities?_sort_desc=name") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert list(reversed(expected)) == rows + + +def test_metadata_sort_desc(app_client): + response = app_client.get("/fixtures/attraction_characteristic") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + assert table["class"] == ["rows-and-columns"] + ths = table.findAll("th") + assert ["pk\xa0▲", "name"] == [th.find("a").string.strip() for th in ths] + rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + expected = [ + [ + '', + '', + ], + [ + '', + '', + ], + ] + assert expected == rows + # Make sure you can reverse that sort order + response = app_client.get("/fixtures/attraction_characteristic?_sort=pk") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + rows = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert list(reversed(expected)) == rows + + +@pytest.mark.parametrize( + "max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url", + ( + ( + 5, + # Default should show 2 facets + "/fixtures/facetable?_facet=_neighborhood", + 2, + True, + "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", + ), + # _facet_size above max_returned_rows should show max_returned_rows (5) + ( + 5, + "/fixtures/facetable?_facet=_neighborhood&_facet_size=50", + 5, + True, + "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", + ), + # If max_returned_rows is high enough, should return all + ( + 20, + "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", + 14, + False, + None, + ), + # If num facets > max_returned_rows, show ... without a link + # _facet_size above max_returned_rows should show max_returned_rows (5) + ( + 5, + "/fixtures/facetable?_facet=_neighborhood&_facet_size=max", + 5, + True, + None, + ), + ), +) +def test_facet_more_links( + max_returned_rows, + path, + expected_num_facets, + expected_ellipses, + expected_ellipses_url, +): + with make_app_client( + settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2} + ) as client: + response = client.get(path) + soup = Soup(response.body, "html.parser") + lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)") + facet_truncated = soup.select_one(".facet-truncated") + assert len(lis) == expected_num_facets + if not expected_ellipses: + assert facet_truncated is None + else: + if expected_ellipses_url: + assert facet_truncated.find("a")["href"] == expected_ellipses_url + else: + assert facet_truncated.find("a") is None + + +def test_unavailable_table_does_not_break_sort_relationships(): + # https://github.com/simonw/datasette/issues/1305 + with make_app_client( + metadata={ + "databases": { + "fixtures": {"tables": {"foreign_key_references": {"allow": False}}} + } + } + ) as client: + response = client.get("/?_sort=relationships") + assert response.status == 200 + + +def test_column_metadata(app_client): + response = app_client.get("/fixtures/roadside_attractions") + soup = Soup(response.body, "html.parser") + dl = soup.find("dl") + assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [ + ("name", "The name of the attraction"), + ("address", "The street address for the attraction"), + ] + assert ( + soup.select("th[data-column=name]")[0]["data-column-description"] + == "The name of the attraction" + ) + assert ( + soup.select("th[data-column=address]")[0]["data-column-description"] + == "The street address for the attraction" + ) + + +@pytest.mark.parametrize("use_facet_size_max", (True, False)) +def test_facet_total_shown_if_facet_max_size(use_facet_size_max): + # https://github.com/simonw/datasette/issues/1423 + with make_app_client(settings={"max_returned_rows": 100}) as client: + path = "/fixtures/sortable?_facet=content&_facet=pk1" + if use_facet_size_max: + path += "&_facet_size=max" + response = client.get(path) + assert response.status == 200 + fragments = ( + '>100', + '8', + ) + for fragment in fragments: + if use_facet_size_max: + assert fragment in response.text + else: + assert fragment not in response.text + + +def test_sort_rowid_with_next(app_client): + # https://github.com/simonw/datasette/issues/1470 + response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid") + assert response.status == 200 + + +def assert_querystring_equal(expected, actual): + assert sorted(expected.split("&")) == sorted(actual.split("&")) diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..972300db --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,24 @@ +def assert_footer_links(soup): + footer_links = soup.find("footer").findAll("a") + assert 4 == len(footer_links) + datasette_link, license_link, source_link, about_link = footer_links + assert "Datasette" == datasette_link.text.strip() + assert "tests/fixtures.py" == source_link.text.strip() + assert "Apache License 2.0" == license_link.text.strip() + assert "About Datasette" == about_link.text.strip() + assert "https://datasette.io/" == datasette_link["href"] + assert ( + "https://github.com/simonw/datasette/blob/main/tests/fixtures.py" + == source_link["href"] + ) + assert ( + "https://github.com/simonw/datasette/blob/main/LICENSE" == license_link["href"] + ) + assert "https://github.com/simonw/datasette" == about_link["href"] + + +def inner_html(soup): + html = str(soup) + # This includes the parent tag - so remove that + inner_html = html.split(">", 1)[1].rsplit("<", 1)[0] + return inner_html.strip() From 492f9835aa7e90540dd0c6324282b109f73df71b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Dec 2021 19:07:19 -0800 Subject: [PATCH 0525/1866] Refactor table view API tests to test_table_api.py Refs #1518 --- tests/test_api.py | 1215 +-------------------------------------- tests/test_table_api.py | 1206 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 1214 insertions(+), 1207 deletions(-) create mode 100644 tests/test_table_api.py diff --git a/tests/test_api.py b/tests/test_api.py index 400dae7e..df9e0fc4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,29 +1,22 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import detect_json1 -from datasette.utils.sqlite import sqlite3, sqlite_version, supports_table_xinfo +from datasette.utils.sqlite import supports_table_xinfo from datasette.version import __version__ from .fixtures import ( # noqa app_client, app_client_no_files, - app_client_with_hash, - app_client_shorter_time_limit, - app_client_larger_cache_size, - app_client_returned_rows_matches_page_size, - app_client_two_attached_databases, - app_client_two_attached_databases_one_immutable, - app_client_conflicting_database_names, - app_client_with_cors, app_client_with_dot, - app_client_with_trace, + app_client_shorter_time_limit, + app_client_two_attached_databases_one_immutable, + app_client_larger_cache_size, + app_client_with_cors, + app_client_two_attached_databases, + app_client_conflicting_database_names, app_client_immutable_and_inspect_file, - generate_compound_rows, - generate_sortable_rows, make_app_client, EXPECTED_PLUGINS, METADATA, ) -import json import pathlib import pytest import sys @@ -680,649 +673,6 @@ def test_invalid_custom_sql(app_client): assert "Statement must be a SELECT" == response.json["error"] -def test_table_json(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects") - assert response.status == 200 - data = response.json - assert ( - data["query"]["sql"] - == "select id, content from simple_primary_key order by id limit 51" - ) - assert data["query"]["params"] == {} - assert data["rows"] == [ - {"id": "1", "content": "hello"}, - {"id": "2", "content": "world"}, - {"id": "3", "content": ""}, - {"id": "4", "content": "RENDER_CELL_DEMO"}, - {"id": "5", "content": "RENDER_CELL_ASYNC"}, - ] - - -def test_table_not_exists_json(app_client): - assert { - "ok": False, - "error": "Table not found: blah", - "status": 404, - "title": None, - } == app_client.get("/fixtures/blah.json").json - - -def test_jsono_redirects_to_shape_objects(app_client_with_hash): - response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono") - response = app_client_with_hash.get(response_1.headers["Location"]) - assert response.status == 302 - assert response.headers["Location"].endswith("?_shape=objects") - - -def test_table_shape_arrays(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays") - assert [ - ["1", "hello"], - ["2", "world"], - ["3", ""], - ["4", "RENDER_CELL_DEMO"], - ["5", "RENDER_CELL_ASYNC"], - ] == response.json["rows"] - - -def test_table_shape_arrayfirst(app_client): - response = app_client.get( - "/fixtures.json?" - + urllib.parse.urlencode( - { - "sql": "select content from simple_primary_key order by id", - "_shape": "arrayfirst", - } - ) - ) - assert [ - "hello", - "world", - "", - "RENDER_CELL_DEMO", - "RENDER_CELL_ASYNC", - ] == response.json - - -def test_table_shape_objects(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects") - assert [ - {"id": "1", "content": "hello"}, - {"id": "2", "content": "world"}, - {"id": "3", "content": ""}, - {"id": "4", "content": "RENDER_CELL_DEMO"}, - {"id": "5", "content": "RENDER_CELL_ASYNC"}, - ] == response.json["rows"] - - -def test_table_shape_array(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=array") - assert [ - {"id": "1", "content": "hello"}, - {"id": "2", "content": "world"}, - {"id": "3", "content": ""}, - {"id": "4", "content": "RENDER_CELL_DEMO"}, - {"id": "5", "content": "RENDER_CELL_ASYNC"}, - ] == response.json - - -def test_table_shape_array_nl(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=array&_nl=on") - lines = response.text.split("\n") - results = [json.loads(line) for line in lines] - assert [ - {"id": "1", "content": "hello"}, - {"id": "2", "content": "world"}, - {"id": "3", "content": ""}, - {"id": "4", "content": "RENDER_CELL_DEMO"}, - {"id": "5", "content": "RENDER_CELL_ASYNC"}, - ] == results - - -def test_table_shape_invalid(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=invalid") - assert { - "ok": False, - "error": "Invalid _shape: invalid", - "status": 400, - "title": None, - } == response.json - - -def test_table_shape_object(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_shape=object") - assert { - "1": {"id": "1", "content": "hello"}, - "2": {"id": "2", "content": "world"}, - "3": {"id": "3", "content": ""}, - "4": {"id": "4", "content": "RENDER_CELL_DEMO"}, - "5": {"id": "5", "content": "RENDER_CELL_ASYNC"}, - } == response.json - - -def test_table_shape_object_compound_primary_key(app_client): - response = app_client.get("/fixtures/compound_primary_key.json?_shape=object") - assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json - - -def test_table_with_slashes_in_name(app_client): - response = app_client.get( - "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" - ) - assert response.status == 200 - data = response.json - assert data["rows"] == [{"pk": "3", "content": "hey"}] - - -def test_table_with_reserved_word_name(app_client): - response = app_client.get("/fixtures/select.json?_shape=objects") - assert response.status == 200 - data = response.json - assert data["rows"] == [ - { - "rowid": 1, - "group": "group", - "having": "having", - "and": "and", - "json": '{"href": "http://example.com/", "label":"Example"}', - } - ] - - -@pytest.mark.parametrize( - "path,expected_rows,expected_pages", - [ - ("/fixtures/no_primary_key.json", 201, 5), - ("/fixtures/paginated_view.json", 201, 9), - ("/fixtures/no_primary_key.json?_size=25", 201, 9), - ("/fixtures/paginated_view.json?_size=50", 201, 5), - ("/fixtures/paginated_view.json?_size=max", 201, 3), - ("/fixtures/123_starts_with_digits.json", 0, 1), - # Ensure faceting doesn't break pagination: - ("/fixtures/compound_three_primary_keys.json?_facet=pk1", 1001, 21), - # Paginating while sorted by an expanded foreign key should work - ( - "/fixtures/roadside_attraction_characteristics.json?_size=2&_sort=attraction_id&_labels=on", - 5, - 3, - ), - ], -) -def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages): - fetched = [] - count = 0 - while path: - response = app_client.get(path) - assert 200 == response.status - count += 1 - fetched.extend(response.json["rows"]) - path = response.json["next_url"] - if path: - assert urllib.parse.urlencode({"_next": response.json["next"]}) in path - path = path.replace("http://localhost", "") - assert count < 30, "Possible infinite loop detected" - - assert expected_rows == len(fetched) - assert expected_pages == count - - -@pytest.mark.parametrize( - "path,expected_error", - [ - ("/fixtures/no_primary_key.json?_size=-4", "_size must be a positive integer"), - ("/fixtures/no_primary_key.json?_size=dog", "_size must be a positive integer"), - ("/fixtures/no_primary_key.json?_size=1001", "_size must be <= 100"), - ], -) -def test_validate_page_size(app_client, path, expected_error): - response = app_client.get(path) - assert expected_error == response.json["error"] - assert 400 == response.status - - -def test_page_size_zero(app_client): - """For _size=0 we return the counts, empty rows and no continuation token""" - response = app_client.get("/fixtures/no_primary_key.json?_size=0") - assert 200 == response.status - assert [] == response.json["rows"] - assert 201 == response.json["filtered_table_rows_count"] - assert None is response.json["next"] - assert None is response.json["next_url"] - - -def test_paginate_compound_keys(app_client): - fetched = [] - path = "/fixtures/compound_three_primary_keys.json?_shape=objects" - page = 0 - while path: - page += 1 - response = app_client.get(path) - fetched.extend(response.json["rows"]) - path = response.json["next_url"] - if path: - path = path.replace("http://localhost", "") - assert page < 100 - assert 1001 == len(fetched) - assert 21 == page - # Should be correctly ordered - contents = [f["content"] for f in fetched] - expected = [r[3] for r in generate_compound_rows(1001)] - assert expected == contents - - -def test_paginate_compound_keys_with_extra_filters(app_client): - fetched = [] - path = ( - "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects" - ) - page = 0 - while path: - page += 1 - assert page < 100 - response = app_client.get(path) - fetched.extend(response.json["rows"]) - path = response.json["next_url"] - if path: - path = path.replace("http://localhost", "") - assert 2 == page - expected = [r[3] for r in generate_compound_rows(1001) if "d" in r[3]] - assert expected == [f["content"] for f in fetched] - - -@pytest.mark.parametrize( - "query_string,sort_key,human_description_en", - [ - ("_sort=sortable", lambda row: row["sortable"], "sorted by sortable"), - ( - "_sort_desc=sortable", - lambda row: -row["sortable"], - "sorted by sortable descending", - ), - ( - "_sort=sortable_with_nulls", - lambda row: ( - 1 if row["sortable_with_nulls"] is not None else 0, - row["sortable_with_nulls"], - ), - "sorted by sortable_with_nulls", - ), - ( - "_sort_desc=sortable_with_nulls", - lambda row: ( - 1 if row["sortable_with_nulls"] is None else 0, - -row["sortable_with_nulls"] - if row["sortable_with_nulls"] is not None - else 0, - row["content"], - ), - "sorted by sortable_with_nulls descending", - ), - # text column contains '$null' - ensure it doesn't confuse pagination: - ("_sort=text", lambda row: row["text"], "sorted by text"), - ], -) -def test_sortable(app_client, query_string, sort_key, human_description_en): - path = f"/fixtures/sortable.json?_shape=objects&{query_string}" - fetched = [] - page = 0 - while path: - page += 1 - assert page < 100 - response = app_client.get(path) - assert human_description_en == response.json["human_description_en"] - fetched.extend(response.json["rows"]) - path = response.json["next_url"] - if path: - path = path.replace("http://localhost", "") - assert 5 == page - expected = list(generate_sortable_rows(201)) - expected.sort(key=sort_key) - assert [r["content"] for r in expected] == [r["content"] for r in fetched] - - -def test_sortable_and_filtered(app_client): - path = ( - "/fixtures/sortable.json" - "?content__contains=d&_sort_desc=sortable&_shape=objects" - ) - response = app_client.get(path) - fetched = response.json["rows"] - assert ( - 'where content contains "d" sorted by sortable descending' - == response.json["human_description_en"] - ) - expected = [row for row in generate_sortable_rows(201) if "d" in row["content"]] - assert len(expected) == response.json["filtered_table_rows_count"] - expected.sort(key=lambda row: -row["sortable"]) - assert [r["content"] for r in expected] == [r["content"] for r in fetched] - - -def test_sortable_argument_errors(app_client): - response = app_client.get("/fixtures/sortable.json?_sort=badcolumn") - assert "Cannot sort table by badcolumn" == response.json["error"] - response = app_client.get("/fixtures/sortable.json?_sort_desc=badcolumn2") - assert "Cannot sort table by badcolumn2" == response.json["error"] - response = app_client.get( - "/fixtures/sortable.json?_sort=sortable_with_nulls&_sort_desc=sortable" - ) - assert "Cannot use _sort and _sort_desc at the same time" == response.json["error"] - - -def test_sortable_columns_metadata(app_client): - response = app_client.get("/fixtures/sortable.json?_sort=content") - assert "Cannot sort table by content" == response.json["error"] - # no_primary_key has ALL sort options disabled - for column in ("content", "a", "b", "c"): - response = app_client.get(f"/fixtures/sortable.json?_sort={column}") - assert f"Cannot sort table by {column}" == response.json["error"] - - -@pytest.mark.parametrize( - "path,expected_rows", - [ - ( - "/fixtures/searchable.json?_search=dog", - [ - [1, "barry cat", "terry dog", "panther"], - [2, "terry dog", "sara weasel", "puma"], - ], - ), - ( - # Special keyword shouldn't break FTS query - "/fixtures/searchable.json?_search=AND", - [], - ), - ( - # Without _searchmode=raw this should return no results - "/fixtures/searchable.json?_search=te*+AND+do*", - [], - ), - ( - # _searchmode=raw - "/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw", - [ - [1, "barry cat", "terry dog", "panther"], - [2, "terry dog", "sara weasel", "puma"], - ], - ), - ( - # _searchmode=raw combined with _search_COLUMN - "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw", - [ - [1, "barry cat", "terry dog", "panther"], - ], - ), - ( - "/fixtures/searchable.json?_search=weasel", - [[2, "terry dog", "sara weasel", "puma"]], - ), - ( - "/fixtures/searchable.json?_search_text2=dog", - [[1, "barry cat", "terry dog", "panther"]], - ), - ( - "/fixtures/searchable.json?_search_name%20with%20.%20and%20spaces=panther", - [[1, "barry cat", "terry dog", "panther"]], - ), - ], -) -def test_searchable(app_client, path, expected_rows): - response = app_client.get(path) - assert expected_rows == response.json["rows"] - - -_SEARCHMODE_RAW_RESULTS = [ - [1, "barry cat", "terry dog", "panther"], - [2, "terry dog", "sara weasel", "puma"], -] - - -@pytest.mark.parametrize( - "table_metadata,querystring,expected_rows", - [ - ( - {}, - "_search=te*+AND+do*", - [], - ), - ( - {"searchmode": "raw"}, - "_search=te*+AND+do*", - _SEARCHMODE_RAW_RESULTS, - ), - ( - {}, - "_search=te*+AND+do*&_searchmode=raw", - _SEARCHMODE_RAW_RESULTS, - ), - # Can be over-ridden with _searchmode=escaped - ( - {"searchmode": "raw"}, - "_search=te*+AND+do*&_searchmode=escaped", - [], - ), - ], -) -def test_searchmode(table_metadata, querystring, expected_rows): - with make_app_client( - metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}} - ) as client: - response = client.get("/fixtures/searchable.json?" + querystring) - assert expected_rows == response.json["rows"] - - -@pytest.mark.parametrize( - "path,expected_rows", - [ - ( - "/fixtures/searchable_view_configured_by_metadata.json?_search=weasel", - [[2, "terry dog", "sara weasel", "puma"]], - ), - # This should return all results because search is not configured: - ( - "/fixtures/searchable_view.json?_search=weasel", - [ - [1, "barry cat", "terry dog", "panther"], - [2, "terry dog", "sara weasel", "puma"], - ], - ), - ( - "/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk", - [[2, "terry dog", "sara weasel", "puma"]], - ), - ], -) -def test_searchable_views(app_client, path, expected_rows): - response = app_client.get(path) - assert expected_rows == response.json["rows"] - - -def test_searchable_invalid_column(app_client): - response = app_client.get("/fixtures/searchable.json?_search_invalid=x") - assert 400 == response.status - assert { - "ok": False, - "error": "Cannot search by that column", - "status": 400, - "title": None, - } == response.json - - -@pytest.mark.parametrize( - "path,expected_rows", - [ - ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]), - ( - "/fixtures/simple_primary_key.json?content__contains=o", - [ - ["1", "hello"], - ["2", "world"], - ["4", "RENDER_CELL_DEMO"], - ], - ), - ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]), - ( - "/fixtures/simple_primary_key.json?content__not=world", - [ - ["1", "hello"], - ["3", ""], - ["4", "RENDER_CELL_DEMO"], - ["5", "RENDER_CELL_ASYNC"], - ], - ), - ], -) -def test_table_filter_queries(app_client, path, expected_rows): - response = app_client.get(path) - assert expected_rows == response.json["rows"] - - -def test_table_filter_queries_multiple_of_same_type(app_client): - response = app_client.get( - "/fixtures/simple_primary_key.json?content__not=world&content__not=hello" - ) - assert [ - ["3", ""], - ["4", "RENDER_CELL_DEMO"], - ["5", "RENDER_CELL_ASYNC"], - ] == response.json["rows"] - - -@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") -def test_table_filter_json_arraycontains(app_client): - response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") - assert response.json["rows"] == [ - [ - 1, - "2019-01-14 08:00:00", - 1, - 1, - "CA", - 1, - "Mission", - '["tag1", "tag2"]', - '[{"foo": "bar"}]', - "one", - ], - [ - 2, - "2019-01-14 08:00:00", - 1, - 1, - "CA", - 1, - "Dogpatch", - '["tag1", "tag3"]', - "[]", - "two", - ], - ] - - -@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") -def test_table_filter_json_arraynotcontains(app_client): - response = app_client.get( - "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]" - ) - assert response.json["rows"] == [ - [ - 1, - "2019-01-14 08:00:00", - 1, - 1, - "CA", - 1, - "Mission", - '["tag1", "tag2"]', - '[{"foo": "bar"}]', - "one", - ] - ] - - -def test_table_filter_extra_where(app_client): - response = app_client.get( - "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'" - ) - assert [ - [ - 2, - "2019-01-14 08:00:00", - 1, - 1, - "CA", - 1, - "Dogpatch", - '["tag1", "tag3"]', - "[]", - "two", - ] - ] == response.json["rows"] - - -def test_table_filter_extra_where_invalid(app_client): - response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'") - assert 400 == response.status - assert "Invalid SQL" == response.json["title"] - - -def test_table_filter_extra_where_disabled_if_no_sql_allowed(): - with make_app_client(metadata={"allow_sql": {}}) as client: - response = client.get( - "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'" - ) - assert 403 == response.status - assert "_where= is not allowed" == response.json["error"] - - -def test_table_through(app_client): - # Just the museums: - response = app_client.get( - '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' - ) - assert [ - [ - 3, - "Burlingame Museum of PEZ Memorabilia", - "214 California Drive, Burlingame, CA 94010", - 37.5793, - -122.3442, - ], - [ - 4, - "Bigfoot Discovery Museum", - "5497 Highway 9, Felton, CA 95018", - 37.0414, - -122.0725, - ], - ] == response.json["rows"] - assert ( - 'where roadside_attraction_characteristics.characteristic_id = "1"' - == response.json["human_description_en"] - ) - - -def test_max_returned_rows(app_client): - response = app_client.get("/fixtures.json?sql=select+content+from+no_primary_key") - data = response.json - assert {"sql": "select content from no_primary_key", "params": {}} == data["query"] - assert data["truncated"] - assert 100 == len(data["rows"]) - - -def test_view(app_client): - response = app_client.get("/fixtures/simple_view.json?_shape=objects") - assert response.status == 200 - data = response.json - assert data["rows"] == [ - {"upper_content": "HELLO", "content": "hello"}, - {"upper_content": "WORLD", "content": "world"}, - {"upper_content": "", "content": ""}, - {"upper_content": "RENDER_CELL_DEMO", "content": "RENDER_CELL_DEMO"}, - {"upper_content": "RENDER_CELL_ASYNC", "content": "RENDER_CELL_ASYNC"}, - ] - - def test_row(app_client): response = app_client.get("/fixtures/simple_primary_key/1.json?_shape=objects") assert response.status == 200 @@ -1390,20 +740,6 @@ def test_row_foreign_key_tables(app_client): ] -def test_unit_filters(app_client): - response = app_client.get( - "/fixtures/units.json?distance__lt=75km&frequency__gt=1kHz" - ) - assert response.status == 200 - data = response.json - - assert data["units"]["distance"] == "m" - assert data["units"]["frequency"] == "Hz" - - assert len(data["rows"]) == 1 - assert data["rows"][0][0] == 2 - - def test_databases_json(app_client_two_attached_databases_one_immutable): response = app_client_two_attached_databases_one_immutable.get("/-/databases.json") databases = response.json @@ -1498,330 +834,6 @@ def test_config_redirects_to_settings(app_client, path, expected_redirect): assert response.headers["Location"] == expected_redirect -def test_page_size_matching_max_returned_rows( - app_client_returned_rows_matches_page_size, -): - fetched = [] - path = "/fixtures/no_primary_key.json" - while path: - response = app_client_returned_rows_matches_page_size.get(path) - fetched.extend(response.json["rows"]) - assert len(response.json["rows"]) in (1, 50) - path = response.json["next_url"] - if path: - path = path.replace("http://localhost", "") - assert 201 == len(fetched) - - -@pytest.mark.parametrize( - "path,expected_facet_results", - [ - ( - "/fixtures/facetable.json?_facet=state&_facet=_city_id", - { - "state": { - "name": "state", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=_city_id", - "results": [ - { - "value": "CA", - "label": "CA", - "count": 10, - "toggle_url": "_facet=state&_facet=_city_id&state=CA", - "selected": False, - }, - { - "value": "MI", - "label": "MI", - "count": 4, - "toggle_url": "_facet=state&_facet=_city_id&state=MI", - "selected": False, - }, - { - "value": "MC", - "label": "MC", - "count": 1, - "toggle_url": "_facet=state&_facet=_city_id&state=MC", - "selected": False, - }, - ], - "truncated": False, - }, - "_city_id": { - "name": "_city_id", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=state", - "results": [ - { - "value": 1, - "label": "San Francisco", - "count": 6, - "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1", - "selected": False, - }, - { - "value": 2, - "label": "Los Angeles", - "count": 4, - "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2", - "selected": False, - }, - { - "value": 3, - "label": "Detroit", - "count": 4, - "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3", - "selected": False, - }, - { - "value": 4, - "label": "Memnonia", - "count": 1, - "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4", - "selected": False, - }, - ], - "truncated": False, - }, - }, - ), - ( - "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI", - { - "state": { - "name": "state", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI", - "results": [ - { - "value": "MI", - "label": "MI", - "count": 4, - "selected": True, - "toggle_url": "_facet=state&_facet=_city_id", - } - ], - "truncated": False, - }, - "_city_id": { - "name": "_city_id", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI", - "results": [ - { - "value": 3, - "label": "Detroit", - "count": 4, - "selected": False, - "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3", - } - ], - "truncated": False, - }, - }, - ), - ( - "/fixtures/facetable.json?_facet=planet_int", - { - "planet_int": { - "name": "planet_int", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json", - "results": [ - { - "value": 1, - "label": 1, - "count": 14, - "selected": False, - "toggle_url": "_facet=planet_int&planet_int=1", - }, - { - "value": 2, - "label": 2, - "count": 1, - "selected": False, - "toggle_url": "_facet=planet_int&planet_int=2", - }, - ], - "truncated": False, - } - }, - ), - ( - # planet_int is an integer field: - "/fixtures/facetable.json?_facet=planet_int&planet_int=1", - { - "planet_int": { - "name": "planet_int", - "hideable": True, - "type": "column", - "toggle_url": "/fixtures/facetable.json?planet_int=1", - "results": [ - { - "value": 1, - "label": 1, - "count": 14, - "selected": True, - "toggle_url": "_facet=planet_int", - } - ], - "truncated": False, - } - }, - ), - ], -) -def test_facets(app_client, path, expected_facet_results): - response = app_client.get(path) - facet_results = response.json["facet_results"] - # We only compare the querystring portion of the taggle_url - for facet_name, facet_info in facet_results.items(): - assert facet_name == facet_info["name"] - assert False is facet_info["truncated"] - for facet_value in facet_info["results"]: - facet_value["toggle_url"] = facet_value["toggle_url"].split("?")[1] - assert expected_facet_results == facet_results - - -def test_suggested_facets(app_client): - suggestions = [ - { - "name": suggestion["name"], - "querystring": suggestion["toggle_url"].split("?")[-1], - } - for suggestion in app_client.get("/fixtures/facetable.json").json[ - "suggested_facets" - ] - ] - expected = [ - {"name": "created", "querystring": "_facet=created"}, - {"name": "planet_int", "querystring": "_facet=planet_int"}, - {"name": "on_earth", "querystring": "_facet=on_earth"}, - {"name": "state", "querystring": "_facet=state"}, - {"name": "_city_id", "querystring": "_facet=_city_id"}, - {"name": "_neighborhood", "querystring": "_facet=_neighborhood"}, - {"name": "tags", "querystring": "_facet=tags"}, - {"name": "complex_array", "querystring": "_facet=complex_array"}, - {"name": "created", "querystring": "_facet_date=created"}, - ] - if detect_json1(): - expected.append({"name": "tags", "querystring": "_facet_array=tags"}) - assert expected == suggestions - - -def test_allow_facet_off(): - with make_app_client(settings={"allow_facet": False}) as client: - assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status - # Should not suggest any facets either: - assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] - - -def test_suggest_facets_off(): - with make_app_client(settings={"suggest_facets": False}) as client: - # Now suggested_facets should be [] - assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] - - -@pytest.mark.parametrize("nofacet", (True, False)) -def test_nofacet(app_client, nofacet): - path = "/fixtures/facetable.json?_facet=state" - if nofacet: - path += "&_nofacet=1" - response = app_client.get(path) - if nofacet: - assert response.json["suggested_facets"] == [] - assert response.json["facet_results"] == {} - else: - assert response.json["suggested_facets"] != [] - assert response.json["facet_results"] != {} - - -@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15))) -def test_nocount(app_client, nocount, expected_count): - path = "/fixtures/facetable.json" - if nocount: - path += "?_nocount=1" - response = app_client.get(path) - assert response.json["filtered_table_rows_count"] == expected_count - - -def test_nocount_nofacet_if_shape_is_object(app_client_with_trace): - response = app_client_with_trace.get( - "/fixtures/facetable.json?_trace=1&_shape=object" - ) - assert "count(*)" not in response.text - - -def test_expand_labels(app_client): - response = app_client.get( - "/fixtures/facetable.json?_shape=object&_labels=1&_size=2" - "&_neighborhood__contains=c" - ) - assert { - "2": { - "pk": 2, - "created": "2019-01-14 08:00:00", - "planet_int": 1, - "on_earth": 1, - "state": "CA", - "_city_id": {"value": 1, "label": "San Francisco"}, - "_neighborhood": "Dogpatch", - "tags": '["tag1", "tag3"]', - "complex_array": "[]", - "distinct_some_null": "two", - }, - "13": { - "pk": 13, - "created": "2019-01-17 08:00:00", - "planet_int": 1, - "on_earth": 1, - "state": "MI", - "_city_id": {"value": 3, "label": "Detroit"}, - "_neighborhood": "Corktown", - "tags": "[]", - "complex_array": "[]", - "distinct_some_null": None, - }, - } == response.json - - -def test_expand_label(app_client): - response = app_client.get( - "/fixtures/foreign_key_references.json?_shape=object" - "&_label=foreign_key_with_label&_size=1" - ) - assert response.json == { - "1": { - "pk": "1", - "foreign_key_with_label": {"value": "1", "label": "hello"}, - "foreign_key_with_blank_label": "3", - "foreign_key_with_no_label": "1", - "foreign_key_compound_pk1": "a", - "foreign_key_compound_pk2": "b", - } - } - - -@pytest.mark.parametrize( - "path,expected_cache_control", - [ - ("/fixtures/facetable.json", "max-age=5"), - ("/fixtures/facetable.json?_ttl=invalid", "max-age=5"), - ("/fixtures/facetable.json?_ttl=10", "max-age=10"), - ("/fixtures/facetable.json?_ttl=0", "no-cache"), - ], -) -def test_ttl_parameter(app_client, path, expected_cache_control): - response = app_client.get(path) - assert expected_cache_control == response.headers["Cache-Control"] - - @pytest.mark.parametrize( "path,expected_redirect", [ @@ -1899,29 +911,6 @@ def test_config_force_https_urls(): assert client.ds._last_request.scheme == "https" -def test_infinity_returned_as_null(app_client): - response = app_client.get("/fixtures/infinity.json?_shape=array") - assert [ - {"rowid": 1, "value": None}, - {"rowid": 2, "value": None}, - {"rowid": 3, "value": 1.5}, - ] == response.json - - -def test_infinity_returned_as_invalid_json_if_requested(app_client): - response = app_client.get("/fixtures/infinity.json?_shape=array&_json_infinity=1") - assert [ - {"rowid": 1, "value": float("inf")}, - {"rowid": 2, "value": float("-inf")}, - {"rowid": 3, "value": 1.5}, - ] == response.json - - -def test_custom_query_with_unicode_characters(app_client): - response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array") - assert [{"id": 1, "name": "San Francisco"}] == response.json - - @pytest.mark.parametrize("trace_debug", (True, False)) def test_trace(trace_debug): with make_app_client(settings={"trace_debug": trace_debug}) as client: @@ -1997,205 +986,17 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): assert db_name == data["database"] -def test_null_and_compound_foreign_keys_are_not_expanded(app_client): - response = app_client.get( - "/fixtures/foreign_key_references.json?_shape=array&_labels=on" - ) - assert response.json == [ - { - "pk": "1", - "foreign_key_with_label": {"value": "1", "label": "hello"}, - "foreign_key_with_blank_label": {"value": "3", "label": ""}, - "foreign_key_with_no_label": {"value": "1", "label": "1"}, - "foreign_key_compound_pk1": "a", - "foreign_key_compound_pk2": "b", - }, - { - "pk": "2", - "foreign_key_with_label": None, - "foreign_key_with_blank_label": None, - "foreign_key_with_no_label": None, - "foreign_key_compound_pk1": None, - "foreign_key_compound_pk2": None, - }, - ] - - def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file): response = app_client_immutable_and_inspect_file.get("/fixtures/sortable.json") assert response.json["filtered_table_rows_count"] == 100 -@pytest.mark.parametrize( - "path,expected_json,expected_text", - [ - ( - "/fixtures/binary_data.json?_shape=array", - [ - {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}}, - {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}}, - {"rowid": 3, "data": None}, - ], - None, - ), - ( - "/fixtures/binary_data.json?_shape=array&_nl=on", - None, - ( - '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n' - '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}\n' - '{"rowid": 3, "data": null}' - ), - ), - ], -) -def test_binary_data_in_json(app_client, path, expected_json, expected_text): - response = app_client.get(path) - if expected_json: - assert response.json == expected_json - else: - assert response.text == expected_text - - -@pytest.mark.parametrize( - "qs", - [ - "", - "?_shape=arrays", - "?_shape=arrayfirst", - "?_shape=object", - "?_shape=objects", - "?_shape=array", - "?_shape=array&_nl=on", - ], -) -def test_paginate_using_link_header(app_client, qs): - path = f"/fixtures/compound_three_primary_keys.json{qs}" - num_pages = 0 - while path: - response = app_client.get(path) - assert response.status == 200 - num_pages += 1 - link = response.headers.get("link") - if link: - assert link.startswith("<") - assert link.endswith('>; rel="next"') - path = link[1:].split(">")[0] - path = path.replace("http://localhost", "") - else: - path = None - assert num_pages == 21 - - -@pytest.mark.skipif( - sqlite_version() < (3, 31, 0), - reason="generated columns were added in SQLite 3.31.0", -) -def test_generated_columns_are_visible_in_datasette(): - with make_app_client( - extra_databases={ - "generated.db": """ - CREATE TABLE generated_columns ( - body TEXT, - id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, - consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED - ); - INSERT INTO generated_columns (body) VALUES ( - '{"number": 1, "string": "This is a string"}' - );""" - } - ) as client: - response = client.get("/generated/generated_columns.json?_shape=array") - assert response.json == [ - { - "rowid": 1, - "body": '{"number": 1, "string": "This is a string"}', - "id": 1, - "consideration": "This is a string", - } - ] - - def test_http_options_request(app_client): response = app_client.request("/fixtures", method="OPTIONS") assert response.status == 200 assert response.text == "ok" -@pytest.mark.parametrize( - "path,expected_columns", - ( - ("/fixtures/facetable.json?_col=created", ["pk", "created"]), - ( - "/fixtures/facetable.json?_nocol=created", - [ - "pk", - "planet_int", - "on_earth", - "state", - "_city_id", - "_neighborhood", - "tags", - "complex_array", - "distinct_some_null", - ], - ), - ( - "/fixtures/facetable.json?_col=state&_col=created", - ["pk", "state", "created"], - ), - ( - "/fixtures/facetable.json?_col=state&_col=state", - ["pk", "state"], - ), - ( - "/fixtures/facetable.json?_col=state&_col=created&_nocol=created", - ["pk", "state"], - ), - ( - # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345 - "/fixtures/facetable.json?_nocol=state&_facet=state", - [ - "pk", - "created", - "planet_int", - "on_earth", - "_city_id", - "_neighborhood", - "tags", - "complex_array", - "distinct_some_null", - ], - ), - ( - "/fixtures/simple_view.json?_nocol=content", - ["upper_content"], - ), - ("/fixtures/simple_view.json?_col=content", ["content"]), - ), -) -def test_col_nocol(app_client, path, expected_columns): - response = app_client.get(path) - assert response.status == 200 - columns = response.json["columns"] - assert columns == expected_columns - - -@pytest.mark.parametrize( - "path,expected_error", - ( - ("/fixtures/facetable.json?_col=bad", "_col=bad - invalid columns"), - ("/fixtures/facetable.json?_nocol=bad", "_nocol=bad - invalid columns"), - ("/fixtures/facetable.json?_nocol=pk", "_nocol=pk - invalid columns"), - ("/fixtures/simple_view.json?_col=bad", "_col=bad - invalid columns"), - ), -) -def test_col_nocol_errors(app_client, path, expected_error): - response = app_client.get(path) - assert response.status == 400 - assert response.json["error"] == expected_error - - @pytest.mark.asyncio async def test_db_path(app_client): db = app_client.ds.get_database() @@ -2205,5 +1006,5 @@ async def test_db_path(app_client): datasette = Datasette([path]) - # this will break with a path + # Previously this broke if path was a pathlib.Path: await datasette.refresh_schemas() diff --git a/tests/test_table_api.py b/tests/test_table_api.py new file mode 100644 index 00000000..a530de44 --- /dev/null +++ b/tests/test_table_api.py @@ -0,0 +1,1206 @@ +from datasette.utils import detect_json1 +from datasette.utils.sqlite import sqlite_version +from .fixtures import ( # noqa + app_client, + app_client_with_hash, + app_client_with_trace, + app_client_returned_rows_matches_page_size, + generate_compound_rows, + generate_sortable_rows, + make_app_client, +) +import json +import pytest +import urllib + + +def test_table_json(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects") + assert response.status == 200 + data = response.json + assert ( + data["query"]["sql"] + == "select id, content from simple_primary_key order by id limit 51" + ) + assert data["query"]["params"] == {} + assert data["rows"] == [ + {"id": "1", "content": "hello"}, + {"id": "2", "content": "world"}, + {"id": "3", "content": ""}, + {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, + ] + + +def test_table_not_exists_json(app_client): + assert { + "ok": False, + "error": "Table not found: blah", + "status": 404, + "title": None, + } == app_client.get("/fixtures/blah.json").json + + +def test_jsono_redirects_to_shape_objects(app_client_with_hash): + response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono") + response = app_client_with_hash.get(response_1.headers["Location"]) + assert response.status == 302 + assert response.headers["Location"].endswith("?_shape=objects") + + +def test_table_shape_arrays(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays") + assert [ + ["1", "hello"], + ["2", "world"], + ["3", ""], + ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], + ] == response.json["rows"] + + +def test_table_shape_arrayfirst(app_client): + response = app_client.get( + "/fixtures.json?" + + urllib.parse.urlencode( + { + "sql": "select content from simple_primary_key order by id", + "_shape": "arrayfirst", + } + ) + ) + assert [ + "hello", + "world", + "", + "RENDER_CELL_DEMO", + "RENDER_CELL_ASYNC", + ] == response.json + + +def test_table_shape_objects(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=objects") + assert [ + {"id": "1", "content": "hello"}, + {"id": "2", "content": "world"}, + {"id": "3", "content": ""}, + {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, + ] == response.json["rows"] + + +def test_table_shape_array(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=array") + assert [ + {"id": "1", "content": "hello"}, + {"id": "2", "content": "world"}, + {"id": "3", "content": ""}, + {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, + ] == response.json + + +def test_table_shape_array_nl(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=array&_nl=on") + lines = response.text.split("\n") + results = [json.loads(line) for line in lines] + assert [ + {"id": "1", "content": "hello"}, + {"id": "2", "content": "world"}, + {"id": "3", "content": ""}, + {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, + ] == results + + +def test_table_shape_invalid(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=invalid") + assert { + "ok": False, + "error": "Invalid _shape: invalid", + "status": 400, + "title": None, + } == response.json + + +def test_table_shape_object(app_client): + response = app_client.get("/fixtures/simple_primary_key.json?_shape=object") + assert { + "1": {"id": "1", "content": "hello"}, + "2": {"id": "2", "content": "world"}, + "3": {"id": "3", "content": ""}, + "4": {"id": "4", "content": "RENDER_CELL_DEMO"}, + "5": {"id": "5", "content": "RENDER_CELL_ASYNC"}, + } == response.json + + +def test_table_shape_object_compound_primary_key(app_client): + response = app_client.get("/fixtures/compound_primary_key.json?_shape=object") + assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json + + +def test_table_with_slashes_in_name(app_client): + response = app_client.get( + "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" + ) + assert response.status == 200 + data = response.json + assert data["rows"] == [{"pk": "3", "content": "hey"}] + + +def test_table_with_reserved_word_name(app_client): + response = app_client.get("/fixtures/select.json?_shape=objects") + assert response.status == 200 + data = response.json + assert data["rows"] == [ + { + "rowid": 1, + "group": "group", + "having": "having", + "and": "and", + "json": '{"href": "http://example.com/", "label":"Example"}', + } + ] + + +@pytest.mark.parametrize( + "path,expected_rows,expected_pages", + [ + ("/fixtures/no_primary_key.json", 201, 5), + ("/fixtures/paginated_view.json", 201, 9), + ("/fixtures/no_primary_key.json?_size=25", 201, 9), + ("/fixtures/paginated_view.json?_size=50", 201, 5), + ("/fixtures/paginated_view.json?_size=max", 201, 3), + ("/fixtures/123_starts_with_digits.json", 0, 1), + # Ensure faceting doesn't break pagination: + ("/fixtures/compound_three_primary_keys.json?_facet=pk1", 1001, 21), + # Paginating while sorted by an expanded foreign key should work + ( + "/fixtures/roadside_attraction_characteristics.json?_size=2&_sort=attraction_id&_labels=on", + 5, + 3, + ), + ], +) +def test_paginate_tables_and_views(app_client, path, expected_rows, expected_pages): + fetched = [] + count = 0 + while path: + response = app_client.get(path) + assert 200 == response.status + count += 1 + fetched.extend(response.json["rows"]) + path = response.json["next_url"] + if path: + assert urllib.parse.urlencode({"_next": response.json["next"]}) in path + path = path.replace("http://localhost", "") + assert count < 30, "Possible infinite loop detected" + + assert expected_rows == len(fetched) + assert expected_pages == count + + +@pytest.mark.parametrize( + "path,expected_error", + [ + ("/fixtures/no_primary_key.json?_size=-4", "_size must be a positive integer"), + ("/fixtures/no_primary_key.json?_size=dog", "_size must be a positive integer"), + ("/fixtures/no_primary_key.json?_size=1001", "_size must be <= 100"), + ], +) +def test_validate_page_size(app_client, path, expected_error): + response = app_client.get(path) + assert expected_error == response.json["error"] + assert 400 == response.status + + +def test_page_size_zero(app_client): + """For _size=0 we return the counts, empty rows and no continuation token""" + response = app_client.get("/fixtures/no_primary_key.json?_size=0") + assert 200 == response.status + assert [] == response.json["rows"] + assert 201 == response.json["filtered_table_rows_count"] + assert None is response.json["next"] + assert None is response.json["next_url"] + + +def test_paginate_compound_keys(app_client): + fetched = [] + path = "/fixtures/compound_three_primary_keys.json?_shape=objects" + page = 0 + while path: + page += 1 + response = app_client.get(path) + fetched.extend(response.json["rows"]) + path = response.json["next_url"] + if path: + path = path.replace("http://localhost", "") + assert page < 100 + assert 1001 == len(fetched) + assert 21 == page + # Should be correctly ordered + contents = [f["content"] for f in fetched] + expected = [r[3] for r in generate_compound_rows(1001)] + assert expected == contents + + +def test_paginate_compound_keys_with_extra_filters(app_client): + fetched = [] + path = ( + "/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects" + ) + page = 0 + while path: + page += 1 + assert page < 100 + response = app_client.get(path) + fetched.extend(response.json["rows"]) + path = response.json["next_url"] + if path: + path = path.replace("http://localhost", "") + assert 2 == page + expected = [r[3] for r in generate_compound_rows(1001) if "d" in r[3]] + assert expected == [f["content"] for f in fetched] + + +@pytest.mark.parametrize( + "query_string,sort_key,human_description_en", + [ + ("_sort=sortable", lambda row: row["sortable"], "sorted by sortable"), + ( + "_sort_desc=sortable", + lambda row: -row["sortable"], + "sorted by sortable descending", + ), + ( + "_sort=sortable_with_nulls", + lambda row: ( + 1 if row["sortable_with_nulls"] is not None else 0, + row["sortable_with_nulls"], + ), + "sorted by sortable_with_nulls", + ), + ( + "_sort_desc=sortable_with_nulls", + lambda row: ( + 1 if row["sortable_with_nulls"] is None else 0, + -row["sortable_with_nulls"] + if row["sortable_with_nulls"] is not None + else 0, + row["content"], + ), + "sorted by sortable_with_nulls descending", + ), + # text column contains '$null' - ensure it doesn't confuse pagination: + ("_sort=text", lambda row: row["text"], "sorted by text"), + ], +) +def test_sortable(app_client, query_string, sort_key, human_description_en): + path = f"/fixtures/sortable.json?_shape=objects&{query_string}" + fetched = [] + page = 0 + while path: + page += 1 + assert page < 100 + response = app_client.get(path) + assert human_description_en == response.json["human_description_en"] + fetched.extend(response.json["rows"]) + path = response.json["next_url"] + if path: + path = path.replace("http://localhost", "") + assert 5 == page + expected = list(generate_sortable_rows(201)) + expected.sort(key=sort_key) + assert [r["content"] for r in expected] == [r["content"] for r in fetched] + + +def test_sortable_and_filtered(app_client): + path = ( + "/fixtures/sortable.json" + "?content__contains=d&_sort_desc=sortable&_shape=objects" + ) + response = app_client.get(path) + fetched = response.json["rows"] + assert ( + 'where content contains "d" sorted by sortable descending' + == response.json["human_description_en"] + ) + expected = [row for row in generate_sortable_rows(201) if "d" in row["content"]] + assert len(expected) == response.json["filtered_table_rows_count"] + expected.sort(key=lambda row: -row["sortable"]) + assert [r["content"] for r in expected] == [r["content"] for r in fetched] + + +def test_sortable_argument_errors(app_client): + response = app_client.get("/fixtures/sortable.json?_sort=badcolumn") + assert "Cannot sort table by badcolumn" == response.json["error"] + response = app_client.get("/fixtures/sortable.json?_sort_desc=badcolumn2") + assert "Cannot sort table by badcolumn2" == response.json["error"] + response = app_client.get( + "/fixtures/sortable.json?_sort=sortable_with_nulls&_sort_desc=sortable" + ) + assert "Cannot use _sort and _sort_desc at the same time" == response.json["error"] + + +def test_sortable_columns_metadata(app_client): + response = app_client.get("/fixtures/sortable.json?_sort=content") + assert "Cannot sort table by content" == response.json["error"] + # no_primary_key has ALL sort options disabled + for column in ("content", "a", "b", "c"): + response = app_client.get(f"/fixtures/sortable.json?_sort={column}") + assert f"Cannot sort table by {column}" == response.json["error"] + + +@pytest.mark.parametrize( + "path,expected_rows", + [ + ( + "/fixtures/searchable.json?_search=dog", + [ + [1, "barry cat", "terry dog", "panther"], + [2, "terry dog", "sara weasel", "puma"], + ], + ), + ( + # Special keyword shouldn't break FTS query + "/fixtures/searchable.json?_search=AND", + [], + ), + ( + # Without _searchmode=raw this should return no results + "/fixtures/searchable.json?_search=te*+AND+do*", + [], + ), + ( + # _searchmode=raw + "/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw", + [ + [1, "barry cat", "terry dog", "panther"], + [2, "terry dog", "sara weasel", "puma"], + ], + ), + ( + # _searchmode=raw combined with _search_COLUMN + "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw", + [ + [1, "barry cat", "terry dog", "panther"], + ], + ), + ( + "/fixtures/searchable.json?_search=weasel", + [[2, "terry dog", "sara weasel", "puma"]], + ), + ( + "/fixtures/searchable.json?_search_text2=dog", + [[1, "barry cat", "terry dog", "panther"]], + ), + ( + "/fixtures/searchable.json?_search_name%20with%20.%20and%20spaces=panther", + [[1, "barry cat", "terry dog", "panther"]], + ), + ], +) +def test_searchable(app_client, path, expected_rows): + response = app_client.get(path) + assert expected_rows == response.json["rows"] + + +_SEARCHMODE_RAW_RESULTS = [ + [1, "barry cat", "terry dog", "panther"], + [2, "terry dog", "sara weasel", "puma"], +] + + +@pytest.mark.parametrize( + "table_metadata,querystring,expected_rows", + [ + ( + {}, + "_search=te*+AND+do*", + [], + ), + ( + {"searchmode": "raw"}, + "_search=te*+AND+do*", + _SEARCHMODE_RAW_RESULTS, + ), + ( + {}, + "_search=te*+AND+do*&_searchmode=raw", + _SEARCHMODE_RAW_RESULTS, + ), + # Can be over-ridden with _searchmode=escaped + ( + {"searchmode": "raw"}, + "_search=te*+AND+do*&_searchmode=escaped", + [], + ), + ], +) +def test_searchmode(table_metadata, querystring, expected_rows): + with make_app_client( + metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}} + ) as client: + response = client.get("/fixtures/searchable.json?" + querystring) + assert expected_rows == response.json["rows"] + + +@pytest.mark.parametrize( + "path,expected_rows", + [ + ( + "/fixtures/searchable_view_configured_by_metadata.json?_search=weasel", + [[2, "terry dog", "sara weasel", "puma"]], + ), + # This should return all results because search is not configured: + ( + "/fixtures/searchable_view.json?_search=weasel", + [ + [1, "barry cat", "terry dog", "panther"], + [2, "terry dog", "sara weasel", "puma"], + ], + ), + ( + "/fixtures/searchable_view.json?_search=weasel&_fts_table=searchable_fts&_fts_pk=pk", + [[2, "terry dog", "sara weasel", "puma"]], + ), + ], +) +def test_searchable_views(app_client, path, expected_rows): + response = app_client.get(path) + assert expected_rows == response.json["rows"] + + +def test_searchable_invalid_column(app_client): + response = app_client.get("/fixtures/searchable.json?_search_invalid=x") + assert 400 == response.status + assert { + "ok": False, + "error": "Cannot search by that column", + "status": 400, + "title": None, + } == response.json + + +@pytest.mark.parametrize( + "path,expected_rows", + [ + ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]), + ( + "/fixtures/simple_primary_key.json?content__contains=o", + [ + ["1", "hello"], + ["2", "world"], + ["4", "RENDER_CELL_DEMO"], + ], + ), + ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]), + ( + "/fixtures/simple_primary_key.json?content__not=world", + [ + ["1", "hello"], + ["3", ""], + ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], + ], + ), + ], +) +def test_table_filter_queries(app_client, path, expected_rows): + response = app_client.get(path) + assert expected_rows == response.json["rows"] + + +def test_table_filter_queries_multiple_of_same_type(app_client): + response = app_client.get( + "/fixtures/simple_primary_key.json?content__not=world&content__not=hello" + ) + assert [ + ["3", ""], + ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], + ] == response.json["rows"] + + +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +def test_table_filter_json_arraycontains(app_client): + response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") + assert response.json["rows"] == [ + [ + 1, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Mission", + '["tag1", "tag2"]', + '[{"foo": "bar"}]', + "one", + ], + [ + 2, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Dogpatch", + '["tag1", "tag3"]', + "[]", + "two", + ], + ] + + +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +def test_table_filter_json_arraynotcontains(app_client): + response = app_client.get( + "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]" + ) + assert response.json["rows"] == [ + [ + 1, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Mission", + '["tag1", "tag2"]', + '[{"foo": "bar"}]', + "one", + ] + ] + + +def test_table_filter_extra_where(app_client): + response = app_client.get( + "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'" + ) + assert [ + [ + 2, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Dogpatch", + '["tag1", "tag3"]', + "[]", + "two", + ] + ] == response.json["rows"] + + +def test_table_filter_extra_where_invalid(app_client): + response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'") + assert 400 == response.status + assert "Invalid SQL" == response.json["title"] + + +def test_table_filter_extra_where_disabled_if_no_sql_allowed(): + with make_app_client(metadata={"allow_sql": {}}) as client: + response = client.get( + "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'" + ) + assert 403 == response.status + assert "_where= is not allowed" == response.json["error"] + + +def test_table_through(app_client): + # Just the museums: + response = app_client.get( + '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + ) + assert [ + [ + 3, + "Burlingame Museum of PEZ Memorabilia", + "214 California Drive, Burlingame, CA 94010", + 37.5793, + -122.3442, + ], + [ + 4, + "Bigfoot Discovery Museum", + "5497 Highway 9, Felton, CA 95018", + 37.0414, + -122.0725, + ], + ] == response.json["rows"] + assert ( + 'where roadside_attraction_characteristics.characteristic_id = "1"' + == response.json["human_description_en"] + ) + + +def test_max_returned_rows(app_client): + response = app_client.get("/fixtures.json?sql=select+content+from+no_primary_key") + data = response.json + assert {"sql": "select content from no_primary_key", "params": {}} == data["query"] + assert data["truncated"] + assert 100 == len(data["rows"]) + + +def test_view(app_client): + response = app_client.get("/fixtures/simple_view.json?_shape=objects") + assert response.status == 200 + data = response.json + assert data["rows"] == [ + {"upper_content": "HELLO", "content": "hello"}, + {"upper_content": "WORLD", "content": "world"}, + {"upper_content": "", "content": ""}, + {"upper_content": "RENDER_CELL_DEMO", "content": "RENDER_CELL_DEMO"}, + {"upper_content": "RENDER_CELL_ASYNC", "content": "RENDER_CELL_ASYNC"}, + ] + + +def test_unit_filters(app_client): + response = app_client.get( + "/fixtures/units.json?distance__lt=75km&frequency__gt=1kHz" + ) + assert response.status == 200 + data = response.json + + assert data["units"]["distance"] == "m" + assert data["units"]["frequency"] == "Hz" + + assert len(data["rows"]) == 1 + assert data["rows"][0][0] == 2 + + +def test_page_size_matching_max_returned_rows( + app_client_returned_rows_matches_page_size, +): + fetched = [] + path = "/fixtures/no_primary_key.json" + while path: + response = app_client_returned_rows_matches_page_size.get(path) + fetched.extend(response.json["rows"]) + assert len(response.json["rows"]) in (1, 50) + path = response.json["next_url"] + if path: + path = path.replace("http://localhost", "") + assert 201 == len(fetched) + + +@pytest.mark.parametrize( + "path,expected_facet_results", + [ + ( + "/fixtures/facetable.json?_facet=state&_facet=_city_id", + { + "state": { + "name": "state", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json?_facet=_city_id", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "_facet=state&_facet=_city_id&state=CA", + "selected": False, + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "_facet=state&_facet=_city_id&state=MI", + "selected": False, + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "_facet=state&_facet=_city_id&state=MC", + "selected": False, + }, + ], + "truncated": False, + }, + "_city_id": { + "name": "_city_id", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json?_facet=state", + "results": [ + { + "value": 1, + "label": "San Francisco", + "count": 6, + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=1", + "selected": False, + }, + { + "value": 2, + "label": "Los Angeles", + "count": 4, + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=2", + "selected": False, + }, + { + "value": 3, + "label": "Detroit", + "count": 4, + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=3", + "selected": False, + }, + { + "value": 4, + "label": "Memnonia", + "count": 1, + "toggle_url": "_facet=state&_facet=_city_id&_city_id__exact=4", + "selected": False, + }, + ], + "truncated": False, + }, + }, + ), + ( + "/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI", + { + "state": { + "name": "state", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json?_facet=_city_id&state=MI", + "results": [ + { + "value": "MI", + "label": "MI", + "count": 4, + "selected": True, + "toggle_url": "_facet=state&_facet=_city_id", + } + ], + "truncated": False, + }, + "_city_id": { + "name": "_city_id", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json?_facet=state&state=MI", + "results": [ + { + "value": 3, + "label": "Detroit", + "count": 4, + "selected": False, + "toggle_url": "_facet=state&_facet=_city_id&state=MI&_city_id__exact=3", + } + ], + "truncated": False, + }, + }, + ), + ( + "/fixtures/facetable.json?_facet=planet_int", + { + "planet_int": { + "name": "planet_int", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json", + "results": [ + { + "value": 1, + "label": 1, + "count": 14, + "selected": False, + "toggle_url": "_facet=planet_int&planet_int=1", + }, + { + "value": 2, + "label": 2, + "count": 1, + "selected": False, + "toggle_url": "_facet=planet_int&planet_int=2", + }, + ], + "truncated": False, + } + }, + ), + ( + # planet_int is an integer field: + "/fixtures/facetable.json?_facet=planet_int&planet_int=1", + { + "planet_int": { + "name": "planet_int", + "hideable": True, + "type": "column", + "toggle_url": "/fixtures/facetable.json?planet_int=1", + "results": [ + { + "value": 1, + "label": 1, + "count": 14, + "selected": True, + "toggle_url": "_facet=planet_int", + } + ], + "truncated": False, + } + }, + ), + ], +) +def test_facets(app_client, path, expected_facet_results): + response = app_client.get(path) + facet_results = response.json["facet_results"] + # We only compare the querystring portion of the taggle_url + for facet_name, facet_info in facet_results.items(): + assert facet_name == facet_info["name"] + assert False is facet_info["truncated"] + for facet_value in facet_info["results"]: + facet_value["toggle_url"] = facet_value["toggle_url"].split("?")[1] + assert expected_facet_results == facet_results + + +def test_suggested_facets(app_client): + suggestions = [ + { + "name": suggestion["name"], + "querystring": suggestion["toggle_url"].split("?")[-1], + } + for suggestion in app_client.get("/fixtures/facetable.json").json[ + "suggested_facets" + ] + ] + expected = [ + {"name": "created", "querystring": "_facet=created"}, + {"name": "planet_int", "querystring": "_facet=planet_int"}, + {"name": "on_earth", "querystring": "_facet=on_earth"}, + {"name": "state", "querystring": "_facet=state"}, + {"name": "_city_id", "querystring": "_facet=_city_id"}, + {"name": "_neighborhood", "querystring": "_facet=_neighborhood"}, + {"name": "tags", "querystring": "_facet=tags"}, + {"name": "complex_array", "querystring": "_facet=complex_array"}, + {"name": "created", "querystring": "_facet_date=created"}, + ] + if detect_json1(): + expected.append({"name": "tags", "querystring": "_facet_array=tags"}) + assert expected == suggestions + + +def test_allow_facet_off(): + with make_app_client(settings={"allow_facet": False}) as client: + assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status + # Should not suggest any facets either: + assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] + + +def test_suggest_facets_off(): + with make_app_client(settings={"suggest_facets": False}) as client: + # Now suggested_facets should be [] + assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] + + +@pytest.mark.parametrize("nofacet", (True, False)) +def test_nofacet(app_client, nofacet): + path = "/fixtures/facetable.json?_facet=state" + if nofacet: + path += "&_nofacet=1" + response = app_client.get(path) + if nofacet: + assert response.json["suggested_facets"] == [] + assert response.json["facet_results"] == {} + else: + assert response.json["suggested_facets"] != [] + assert response.json["facet_results"] != {} + + +@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15))) +def test_nocount(app_client, nocount, expected_count): + path = "/fixtures/facetable.json" + if nocount: + path += "?_nocount=1" + response = app_client.get(path) + assert response.json["filtered_table_rows_count"] == expected_count + + +def test_nocount_nofacet_if_shape_is_object(app_client_with_trace): + response = app_client_with_trace.get( + "/fixtures/facetable.json?_trace=1&_shape=object" + ) + assert "count(*)" not in response.text + + +def test_expand_labels(app_client): + response = app_client.get( + "/fixtures/facetable.json?_shape=object&_labels=1&_size=2" + "&_neighborhood__contains=c" + ) + assert { + "2": { + "pk": 2, + "created": "2019-01-14 08:00:00", + "planet_int": 1, + "on_earth": 1, + "state": "CA", + "_city_id": {"value": 1, "label": "San Francisco"}, + "_neighborhood": "Dogpatch", + "tags": '["tag1", "tag3"]', + "complex_array": "[]", + "distinct_some_null": "two", + }, + "13": { + "pk": 13, + "created": "2019-01-17 08:00:00", + "planet_int": 1, + "on_earth": 1, + "state": "MI", + "_city_id": {"value": 3, "label": "Detroit"}, + "_neighborhood": "Corktown", + "tags": "[]", + "complex_array": "[]", + "distinct_some_null": None, + }, + } == response.json + + +def test_expand_label(app_client): + response = app_client.get( + "/fixtures/foreign_key_references.json?_shape=object" + "&_label=foreign_key_with_label&_size=1" + ) + assert response.json == { + "1": { + "pk": "1", + "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_blank_label": "3", + "foreign_key_with_no_label": "1", + "foreign_key_compound_pk1": "a", + "foreign_key_compound_pk2": "b", + } + } + + +@pytest.mark.parametrize( + "path,expected_cache_control", + [ + ("/fixtures/facetable.json", "max-age=5"), + ("/fixtures/facetable.json?_ttl=invalid", "max-age=5"), + ("/fixtures/facetable.json?_ttl=10", "max-age=10"), + ("/fixtures/facetable.json?_ttl=0", "no-cache"), + ], +) +def test_ttl_parameter(app_client, path, expected_cache_control): + response = app_client.get(path) + assert expected_cache_control == response.headers["Cache-Control"] + + +def test_infinity_returned_as_null(app_client): + response = app_client.get("/fixtures/infinity.json?_shape=array") + assert [ + {"rowid": 1, "value": None}, + {"rowid": 2, "value": None}, + {"rowid": 3, "value": 1.5}, + ] == response.json + + +def test_infinity_returned_as_invalid_json_if_requested(app_client): + response = app_client.get("/fixtures/infinity.json?_shape=array&_json_infinity=1") + assert [ + {"rowid": 1, "value": float("inf")}, + {"rowid": 2, "value": float("-inf")}, + {"rowid": 3, "value": 1.5}, + ] == response.json + + +def test_custom_query_with_unicode_characters(app_client): + response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array") + assert [{"id": 1, "name": "San Francisco"}] == response.json + + +def test_null_and_compound_foreign_keys_are_not_expanded(app_client): + response = app_client.get( + "/fixtures/foreign_key_references.json?_shape=array&_labels=on" + ) + assert response.json == [ + { + "pk": "1", + "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_blank_label": {"value": "3", "label": ""}, + "foreign_key_with_no_label": {"value": "1", "label": "1"}, + "foreign_key_compound_pk1": "a", + "foreign_key_compound_pk2": "b", + }, + { + "pk": "2", + "foreign_key_with_label": None, + "foreign_key_with_blank_label": None, + "foreign_key_with_no_label": None, + "foreign_key_compound_pk1": None, + "foreign_key_compound_pk2": None, + }, + ] + + +@pytest.mark.parametrize( + "path,expected_json,expected_text", + [ + ( + "/fixtures/binary_data.json?_shape=array", + [ + {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}}, + {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}}, + {"rowid": 3, "data": None}, + ], + None, + ), + ( + "/fixtures/binary_data.json?_shape=array&_nl=on", + None, + ( + '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n' + '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}\n' + '{"rowid": 3, "data": null}' + ), + ), + ], +) +def test_binary_data_in_json(app_client, path, expected_json, expected_text): + response = app_client.get(path) + if expected_json: + assert response.json == expected_json + else: + assert response.text == expected_text + + +@pytest.mark.parametrize( + "qs", + [ + "", + "?_shape=arrays", + "?_shape=arrayfirst", + "?_shape=object", + "?_shape=objects", + "?_shape=array", + "?_shape=array&_nl=on", + ], +) +def test_paginate_using_link_header(app_client, qs): + path = f"/fixtures/compound_three_primary_keys.json{qs}" + num_pages = 0 + while path: + response = app_client.get(path) + assert response.status == 200 + num_pages += 1 + link = response.headers.get("link") + if link: + assert link.startswith("<") + assert link.endswith('>; rel="next"') + path = link[1:].split(">")[0] + path = path.replace("http://localhost", "") + else: + path = None + assert num_pages == 21 + + +@pytest.mark.skipif( + sqlite_version() < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +def test_generated_columns_are_visible_in_datasette(): + with make_app_client( + extra_databases={ + "generated.db": """ + CREATE TABLE generated_columns ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED + ); + INSERT INTO generated_columns (body) VALUES ( + '{"number": 1, "string": "This is a string"}' + );""" + } + ) as client: + response = client.get("/generated/generated_columns.json?_shape=array") + assert response.json == [ + { + "rowid": 1, + "body": '{"number": 1, "string": "This is a string"}', + "id": 1, + "consideration": "This is a string", + } + ] + + +@pytest.mark.parametrize( + "path,expected_columns", + ( + ("/fixtures/facetable.json?_col=created", ["pk", "created"]), + ( + "/fixtures/facetable.json?_nocol=created", + [ + "pk", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + ), + ( + "/fixtures/facetable.json?_col=state&_col=created", + ["pk", "state", "created"], + ), + ( + "/fixtures/facetable.json?_col=state&_col=state", + ["pk", "state"], + ), + ( + "/fixtures/facetable.json?_col=state&_col=created&_nocol=created", + ["pk", "state"], + ), + ( + # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345 + "/fixtures/facetable.json?_nocol=state&_facet=state", + [ + "pk", + "created", + "planet_int", + "on_earth", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + ), + ( + "/fixtures/simple_view.json?_nocol=content", + ["upper_content"], + ), + ("/fixtures/simple_view.json?_col=content", ["content"]), + ), +) +def test_col_nocol(app_client, path, expected_columns): + response = app_client.get(path) + assert response.status == 200 + columns = response.json["columns"] + assert columns == expected_columns + + +@pytest.mark.parametrize( + "path,expected_error", + ( + ("/fixtures/facetable.json?_col=bad", "_col=bad - invalid columns"), + ("/fixtures/facetable.json?_nocol=bad", "_nocol=bad - invalid columns"), + ("/fixtures/facetable.json?_nocol=pk", "_nocol=pk - invalid columns"), + ("/fixtures/simple_view.json?_col=bad", "_col=bad - invalid columns"), + ), +) +def test_col_nocol_errors(app_client, path, expected_error): + response = app_client.get(path) + assert response.status == 400 + assert response.json["error"] == expected_error From a6ff123de5464806441f6a6f95145c9a83b7f20b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Dec 2021 12:01:51 -0800 Subject: [PATCH 0526/1866] keep_blank_values=True when parsing query_string, closes #1551 Refs #1518 --- datasette/utils/asgi.py | 2 +- datasette/views/table.py | 11 +++-------- tests/test_internals_request.py | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index ad137fa9..cd3ec654 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -97,7 +97,7 @@ class Request: @property def args(self): - return MultiParams(parse_qs(qs=self.query_string)) + return MultiParams(parse_qs(qs=self.query_string, keep_blank_values=True)) @property def actor(self): diff --git a/datasette/views/table.py b/datasette/views/table.py index f58b78f5..59010723 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -393,21 +393,16 @@ class TableView(RowTableShared): nocount = True nofacet = True - # Ensure we don't drop anything with an empty value e.g. ?name__exact= - args = MultiParams( - urllib.parse.parse_qs(request.query_string, keep_blank_values=True) - ) - # Special args start with _ and do not contain a __ # That's so if there is a column that starts with _ # it can still be queried using ?_col__exact=blah special_args = {} other_args = [] - for key in args: + for key in request.args: if key.startswith("_") and "__" not in key: - special_args[key] = args[key] + special_args[key] = request.args[key] else: - for v in args.getlist(key): + for v in request.args.getlist(key): other_args.append((key, v)) # Handle ?_filter_column and redirect, if present diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index cd956f3f..01c93eec 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -121,3 +121,19 @@ def test_request_properties(path, query_string, expected_full_path): assert request.path == path assert request.query_string == query_string assert request.full_path == expected_full_path + + +def test_request_blank_values(): + query_string = "a=b&foo=bar&foo=bar2&baz=" + path_with_query_string = "/?" + query_string + scope = { + "http_version": "1.1", + "method": "POST", + "path": "/", + "raw_path": path_with_query_string.encode("latin-1"), + "query_string": query_string.encode("latin-1"), + "scheme": "http", + "type": "http", + } + request = Request(scope, None) + assert request.args._data == {"a": ["b"], "foo": ["bar", "bar2"], "baz": [""]} From 8b411a6b70e93e044820d613a28607ba5d6fe416 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 15:22:21 -0800 Subject: [PATCH 0527/1866] Update pytest-xdist requirement from <2.5,>=2.2.1 to >=2.2.1,<2.6 (#1548) Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.2.1...v2.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9b5bab61..da8dea49 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "test": [ "pytest>=5.2.2,<6.3.0", - "pytest-xdist>=2.2.1,<2.5", + "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.10,<0.17", "beautifulsoup4>=4.8.1,<4.11.0", "black==21.11b1", From f5538e7161cce92a4dfaa7c5b71fcb6755d96c05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 15:22:29 -0800 Subject: [PATCH 0528/1866] Bump black from 21.11b1 to 21.12b0 (#1543) Bumps [black](https://github.com/psf/black) from 21.11b1 to 21.12b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index da8dea49..534265c2 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.10,<0.17", "beautifulsoup4>=4.8.1,<4.11.0", - "black==21.11b1", + "black==21.12b0", "pytest-timeout>=1.4.2,<2.1", "trustme>=0.7,<0.10", ], From 4f02c8d4d7f8672cc98e5f8d435b5dc8fb5211dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Dec 2021 12:28:34 -0800 Subject: [PATCH 0529/1866] Test for JSON in query_string name, refs #621 Plus simplified implementation of test_request_blank_values --- tests/test_internals_request.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 01c93eec..44aaa153 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -124,16 +124,18 @@ def test_request_properties(path, query_string, expected_full_path): def test_request_blank_values(): - query_string = "a=b&foo=bar&foo=bar2&baz=" - path_with_query_string = "/?" + query_string - scope = { - "http_version": "1.1", - "method": "POST", - "path": "/", - "raw_path": path_with_query_string.encode("latin-1"), - "query_string": query_string.encode("latin-1"), - "scheme": "http", - "type": "http", - } - request = Request(scope, None) + request = Request.fake("/?a=b&foo=bar&foo=bar2&baz=") assert request.args._data == {"a": ["b"], "foo": ["bar", "bar2"], "baz": [""]} + + +def test_json_in_query_string_name(): + query_string = ( + '?_through.["roadside_attraction_characteristics"%2C"characteristic_id"]=1' + ) + request = Request.fake("/" + query_string) + assert ( + request.args[ + '_through.["roadside_attraction_characteristics","characteristic_id"]' + ] + == "1" + ) From eb53837d2aeacaffd8d37f81a6639139c6a0b4d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Dec 2021 09:58:01 -0800 Subject: [PATCH 0530/1866] Always show count of distinct facet values, closes #1556 Refs #1423 --- datasette/templates/table.html | 2 +- datasette/views/table.py | 1 - tests/test_table_html.py | 13 ++++--------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 4b9df8e1..f3749b57 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -157,7 +157,7 @@

    {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} - {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %} + {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }} {% if facet_info.hideable %} ✖ diff --git a/datasette/views/table.py b/datasette/views/table.py index 59010723..bb5876cc 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -937,7 +937,6 @@ class TableView(RowTableShared): key=lambda f: (len(f["results"]), f["name"]), reverse=True, ), - "show_facet_counts": special_args.get("_facet_size") == "max", "extra_wheres_for_ui": extra_wheres_for_ui, "form_hidden_args": form_hidden_args, "is_sortable": any(c["sortable"] for c in display_columns), diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2fbb53bd..50d679a0 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1015,24 +1015,19 @@ def test_column_metadata(app_client): ) -@pytest.mark.parametrize("use_facet_size_max", (True, False)) -def test_facet_total_shown_if_facet_max_size(use_facet_size_max): +def test_facet_total(): # https://github.com/simonw/datasette/issues/1423 + # https://github.com/simonw/datasette/issues/1556 with make_app_client(settings={"max_returned_rows": 100}) as client: path = "/fixtures/sortable?_facet=content&_facet=pk1" - if use_facet_size_max: - path += "&_facet_size=max" response = client.get(path) assert response.status == 200 fragments = ( - '>100', + '>30', '8', ) for fragment in fragments: - if use_facet_size_max: - assert fragment in response.text - else: - assert fragment not in response.text + assert fragment in response.text def test_sort_rowid_with_next(app_client): From 40e5b0a5b5cbbe7ec9b1a525d61f58227061597e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 10:03:10 -0800 Subject: [PATCH 0531/1866] How to create indexes with sqlite-utils --- docs/facets.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/facets.rst b/docs/facets.rst index 7730e4ac..4bbfa16f 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -133,6 +133,10 @@ The performance of facets can be greatly improved by adding indexes on the colum Enter ".help" for usage hints. sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state"); +Or using the `sqlite-utils `__ command-line utility:: + + $ sqlite-utils create-index mydatabase.db Food_Trucks state + .. _facet_by_json_array: Facet by JSON array From 20a2ed6bec367d2f6759be4a879364a72780b59d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 10:47:22 -0800 Subject: [PATCH 0532/1866] Fixed bug with metadata config of array/date facets, closes #1552 Thanks @davidbgk for spotting the fix for the bug. --- datasette/facets.py | 2 +- docs/facets.rst | 20 ++++++++++++++++++-- tests/test_facets.py | 25 ++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 8fd2177a..51fccb01 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -30,7 +30,7 @@ def load_facet_configs(request, table_metadata): assert ( len(metadata_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, metadata_config = metadata_config.items()[0] + type, metadata_config = list(metadata_config.items())[0] if isinstance(metadata_config, str): metadata_config = {"simple": metadata_config} facet_configs.setdefault(type, []).append( diff --git a/docs/facets.rst b/docs/facets.rst index 4bbfa16f..0228aa84 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -16,7 +16,9 @@ To turn on faceting for specific columns on a Datasette table view, add one or m /dbname/tablename?_facet=state&_facet=city_id -This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this:: +This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this: + +.. code-block:: json { "state": { @@ -93,7 +95,9 @@ Facets in metadata.json You can turn facets on by default for specific tables by adding them to a ``"facets"`` key in a Datasette :ref:`metadata` file. -Here's an example that turns on faceting by default for the ``qLegalStatus`` column in the ``Street_Tree_List`` table in the ``sf-trees`` database:: +Here's an example that turns on faceting by default for the ``qLegalStatus`` column in the ``Street_Tree_List`` table in the ``sf-trees`` database: + +.. code-block:: json { "databases": { @@ -109,6 +113,18 @@ Here's an example that turns on faceting by default for the ``qLegalStatus`` col Facets defined in this way will always be shown in the interface and returned in the API, regardless of the ``_facet`` arguments passed to the view. +You can specify :ref:`array ` or :ref:`date ` facets in metadata using JSON objects with a single key of ``array`` or ``date`` and a value specifying the column, like this: + +.. code-block:: json + + { + "facets": [ + {"array": "tags"}, + {"date": "created"} + ] + } + + Suggested facets ---------------- diff --git a/tests/test_facets.py b/tests/test_facets.py index 429117cb..5b1aa935 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -3,7 +3,7 @@ from datasette.database import Database from datasette.facets import ColumnFacet, ArrayFacet, DateFacet from datasette.utils.asgi import Request from datasette.utils import detect_json1 -from .fixtures import app_client # noqa +from .fixtures import app_client, make_app_client # noqa import json import pytest @@ -588,3 +588,26 @@ async def test_facet_size(): ) data5 = response5.json() assert len(data5["facet_results"]["city"]["results"]) == 20 + + +def test_other_types_of_facet_in_metadata(): + with make_app_client( + metadata={ + "databases": { + "fixtures": { + "tables": { + "facetable": { + "facets": ["state", {"array": "tags"}, {"date": "created"}] + } + } + } + } + } + ) as client: + response = client.get("/fixtures/facetable") + for fragment in ( + "created (date)\n", + "tags (array)\n", + "state\n", + ): + assert fragment in response.text From 992496f2611a72bd51e94bfd0b17c1d84e732487 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 11:24:54 -0800 Subject: [PATCH 0533/1866] ?_nosuggest=1 parameter for table views, closes #1557 --- datasette/views/table.py | 2 ++ docs/json_api.rst | 3 +++ tests/test_table_api.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/datasette/views/table.py b/datasette/views/table.py index bb5876cc..f294ffb1 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -388,6 +388,7 @@ class TableView(RowTableShared): nocount = request.args.get("_nocount") nofacet = request.args.get("_nofacet") + nosuggest = request.args.get("_nosuggest") if request.args.get("_shape") in ("array", "object"): nocount = True @@ -846,6 +847,7 @@ class TableView(RowTableShared): and self.ds.setting("allow_facet") and not _next and not nofacet + and not nosuggest ): for facet in facet_instances: suggested_facets.extend(await facet.suggest()) diff --git a/docs/json_api.rst b/docs/json_api.rst index 7d3123b7..bd55c163 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -397,6 +397,9 @@ Special table arguments ``?_nofacet=1`` Disable all facets and facet suggestions for this page, including any defined by :ref:`facets_metadata`. +``?_nosuggest=1`` + Disable facet suggestions for this page. + ``?_nocount=1`` Disable the ``select count(*)`` query used on this page - a count of ``None`` will be returned instead. diff --git a/tests/test_table_api.py b/tests/test_table_api.py index a530de44..6a6daed5 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -915,6 +915,21 @@ def test_nofacet(app_client, nofacet): assert response.json["facet_results"] != {} +@pytest.mark.parametrize("nosuggest", (True, False)) +def test_nosuggest(app_client, nosuggest): + path = "/fixtures/facetable.json?_facet=state" + if nosuggest: + path += "&_nosuggest=1" + response = app_client.get(path) + if nosuggest: + assert response.json["suggested_facets"] == [] + # But facets should still be returned: + assert response.json["facet_results"] != {} + else: + assert response.json["suggested_facets"] != [] + assert response.json["facet_results"] != {} + + @pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15))) def test_nocount(app_client, nocount, expected_count): path = "/fixtures/facetable.json" From 95d0dd7a1cf6be6b7da41e1404184217eb93f64a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 12:12:04 -0800 Subject: [PATCH 0534/1866] Fix for colliding facet types bug, closes #625 Refs #830 --- datasette/facets.py | 74 ++++++++++++++++++++++------------------ datasette/views/table.py | 9 ++++- docs/plugin_hooks.rst | 6 ++-- tests/test_facets.py | 30 ++++++++-------- 4 files changed, 67 insertions(+), 52 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 51fccb01..a1bb4a5f 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -193,7 +193,7 @@ class ColumnFacet(Facet): return suggested_facets async def facet_results(self): - facet_results = {} + facet_results = [] facets_timed_out = [] qs_pairs = self.get_querystring_pairs() @@ -221,16 +221,18 @@ class ColumnFacet(Facet): custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] - facet_results[column] = { - "name": column, - "type": self.type, - "hideable": source != "metadata", - "toggle_url": self.ds.urls.path( - path_with_removed_args(self.request, {"_facet": column}) - ), - "results": facet_results_values, - "truncated": len(facet_rows_results) > facet_size, - } + facet_results.append( + { + "name": column, + "type": self.type, + "hideable": source != "metadata", + "toggle_url": self.ds.urls.path( + path_with_removed_args(self.request, {"_facet": column}) + ), + "results": facet_results_values, + "truncated": len(facet_rows_results) > facet_size, + } + ) facet_rows = facet_rows_results.rows[:facet_size] if self.table: # Attempt to expand foreign keys into labels @@ -352,7 +354,7 @@ class ArrayFacet(Facet): async def facet_results(self): # self.configs should be a plain list of columns - facet_results = {} + facet_results = [] facets_timed_out = [] facet_size = self.get_facet_size() @@ -392,16 +394,20 @@ class ArrayFacet(Facet): custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] - facet_results[column] = { - "name": column, - "type": self.type, - "results": facet_results_values, - "hideable": source != "metadata", - "toggle_url": self.ds.urls.path( - path_with_removed_args(self.request, {"_facet_array": column}) - ), - "truncated": len(facet_rows_results) > facet_size, - } + facet_results.append( + { + "name": column, + "type": self.type, + "results": facet_results_values, + "hideable": source != "metadata", + "toggle_url": self.ds.urls.path( + path_with_removed_args( + self.request, {"_facet_array": column} + ) + ), + "truncated": len(facet_rows_results) > facet_size, + } + ) facet_rows = facet_rows_results.rows[:facet_size] pairs = self.get_querystring_pairs() for row in facet_rows: @@ -480,7 +486,7 @@ class DateFacet(Facet): return suggested_facets async def facet_results(self): - facet_results = {} + facet_results = [] facets_timed_out = [] args = dict(self.get_querystring_pairs()) facet_size = self.get_facet_size() @@ -507,16 +513,18 @@ class DateFacet(Facet): custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] - facet_results[column] = { - "name": column, - "type": self.type, - "results": facet_results_values, - "hideable": source != "metadata", - "toggle_url": path_with_removed_args( - self.request, {"_facet_date": column} - ), - "truncated": len(facet_rows_results) > facet_size, - } + facet_results.append( + { + "name": column, + "type": self.type, + "results": facet_results_values, + "hideable": source != "metadata", + "toggle_url": path_with_removed_args( + self.request, {"_facet_date": column} + ), + "truncated": len(facet_rows_results) > facet_size, + } + ) facet_rows = facet_rows_results.rows[:facet_size] for row in facet_rows: selected = str(args.get(f"{column}__date")) == str(row["value"]) diff --git a/datasette/views/table.py b/datasette/views/table.py index f294ffb1..3d0e27cb 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -754,7 +754,14 @@ class TableView(RowTableShared): instance_facet_results, instance_facets_timed_out, ) = await facet.facet_results() - facet_results.update(instance_facet_results) + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info facets_timed_out.extend(instance_facets_timed_out) # Figure out columns and rows for the query diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 23f19e38..4a7c36c3 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -668,7 +668,7 @@ Each Facet subclass implements a new type of facet operation. The class should l async def facet_results(self): # This should execute the facet operation and return results, again # using self.sql and self.params as the starting point - facet_results = {} + facet_results = [] facets_timed_out = [] facet_size = self.get_facet_size() # Do some calculations here... @@ -683,11 +683,11 @@ Each Facet subclass implements a new type of facet operation. The class should l "toggle_url": self.ds.absolute_url(self.request, toggle_path), "selected": selected, }) - facet_results[column] = { + facet_results.append({ "name": column, "results": facet_results_values, "truncated": len(facet_rows_results) > facet_size, - } + }) except QueryInterrupted: facets_timed_out.append(column) diff --git a/tests/test_facets.py b/tests/test_facets.py index 5b1aa935..a99979d3 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -107,8 +107,8 @@ async def test_column_facet_results(app_client): ) buckets, timed_out = await facet.facet_results() assert [] == timed_out - assert { - "_city_id": { + assert [ + { "name": "_city_id", "type": "column", "hideable": True, @@ -145,7 +145,7 @@ async def test_column_facet_results(app_client): ], "truncated": False, } - } == buckets + ] == buckets @pytest.mark.asyncio @@ -159,8 +159,8 @@ async def test_column_facet_results_column_starts_with_underscore(app_client): ) buckets, timed_out = await facet.facet_results() assert [] == timed_out - assert buckets == { - "_neighborhood": { + assert buckets == [ + { "name": "_neighborhood", "type": "column", "hideable": True, @@ -267,7 +267,7 @@ async def test_column_facet_results_column_starts_with_underscore(app_client): ], "truncated": False, } - } + ] @pytest.mark.asyncio @@ -282,8 +282,8 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client): ) buckets, timed_out = await facet.facet_results() assert [] == timed_out - assert { - "_city_id": { + assert [ + { "name": "_city_id", "type": "column", "hideable": False, @@ -320,7 +320,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client): ], "truncated": False, } - } == buckets + ] == buckets @pytest.mark.asyncio @@ -369,8 +369,8 @@ async def test_array_facet_results(app_client): ) buckets, timed_out = await facet.facet_results() assert [] == timed_out - assert { - "tags": { + assert [ + { "name": "tags", "type": "array", "results": [ @@ -400,7 +400,7 @@ async def test_array_facet_results(app_client): "toggle_url": "/", "truncated": False, } - } == buckets + ] == buckets @pytest.mark.asyncio @@ -471,8 +471,8 @@ async def test_date_facet_results(app_client): ) buckets, timed_out = await facet.facet_results() assert [] == timed_out - assert { - "created": { + assert [ + { "name": "created", "type": "date", "results": [ @@ -509,7 +509,7 @@ async def test_date_facet_results(app_client): "toggle_url": "/", "truncated": False, } - } == buckets + ] == buckets @pytest.mark.asyncio From 0d4145d0f4d8b2a7edc1ba4aac1be56cd536a10a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 12:30:31 -0800 Subject: [PATCH 0535/1866] Additional test for #625 --- tests/test_facets.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_facets.py b/tests/test_facets.py index a99979d3..3f292a3b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -611,3 +611,16 @@ def test_other_types_of_facet_in_metadata(): "state\n", ): assert fragment in response.text + + +def test_conflicting_facet_names_json(app_client): + response = app_client.get( + "/fixtures/facetable.json?_facet=created&_facet_date=created" + "&_facet=tags&_facet_array=tags" + ) + assert set(response.json["facet_results"].keys()) == { + "created", + "tags", + "created_2", + "tags_2", + } From 2c07327d23d9c5cf939ada9ba4091c1b8b2ba42d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 13:43:44 -0800 Subject: [PATCH 0536/1866] Move columns_to_select to TableView class, add lots of comments, refs #1518 --- datasette/views/table.py | 117 +++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 3d0e27cb..d37a3066 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -64,41 +64,6 @@ class Row: class RowTableShared(DataView): - async def columns_to_select(self, db, table, request): - table_columns = await db.table_columns(table) - pks = await db.primary_keys(table) - columns = list(table_columns) - if "_col" in request.args: - columns = list(pks) - _cols = request.args.getlist("_col") - bad_columns = [column for column in _cols if column not in table_columns] - if bad_columns: - raise DatasetteError( - "_col={} - invalid columns".format(", ".join(bad_columns)), - status=400, - ) - # De-duplicate maintaining order: - columns.extend(dict.fromkeys(_cols)) - if "_nocol" in request.args: - # Return all columns EXCEPT these - bad_columns = [ - column - for column in request.args.getlist("_nocol") - if (column not in table_columns) or (column in pks) - ] - if bad_columns: - raise DatasetteError( - "_nocol={} - invalid columns".format(", ".join(bad_columns)), - status=400, - ) - tmp_columns = [ - column - for column in columns - if column not in request.args.getlist("_nocol") - ] - columns = tmp_columns - return columns - async def sortable_columns_for_table(self, database, table, use_rowid): db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) @@ -321,6 +286,39 @@ class TableView(RowTableShared): write=bool(canned_query.get("write")), ) + async def columns_to_select(self, table_columns, pks, request): + columns = list(table_columns) + if "_col" in request.args: + columns = list(pks) + _cols = request.args.getlist("_col") + bad_columns = [column for column in _cols if column not in table_columns] + if bad_columns: + raise DatasetteError( + "_col={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + # De-duplicate maintaining order: + columns.extend(dict.fromkeys(_cols)) + if "_nocol" in request.args: + # Return all columns EXCEPT these + bad_columns = [ + column + for column in request.args.getlist("_nocol") + if (column not in table_columns) or (column in pks) + ] + if bad_columns: + raise DatasetteError( + "_nocol={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + tmp_columns = [ + column + for column in columns + if column not in request.args.getlist("_nocol") + ] + columns = tmp_columns + return columns + async def data( self, request, @@ -331,6 +329,7 @@ class TableView(RowTableShared): _next=None, _size=None, ): + # If this is a canned query, not a table, then dispatch to QueryView instead canned_query = await self.ds.get_canned_query(database, table, request.actor) if canned_query: return await QueryView(self.ds).data( @@ -348,9 +347,12 @@ class TableView(RowTableShared): db = self.ds.databases[database] is_view = bool(await db.get_view_definition(table)) table_exists = bool(await db.table_exists(table)) + + # If table or view not found, return 404 if not is_view and not table_exists: raise NotFound(f"Table not found: {table}") + # Ensure user has permission to view this table await self.check_permissions( request, [ @@ -364,15 +366,18 @@ class TableView(RowTableShared): None, "view-table", (database, table), default=True ) + # Introspect columns and primary keys for table pks = await db.primary_keys(table) table_columns = await db.table_columns(table) - specified_columns = await self.columns_to_select(db, table, request) + # Take ?_col= and ?_nocol= into account + specified_columns = await self.columns_to_select(table_columns, pks, request) select_specified_columns = ", ".join( escape_sqlite(t) for t in specified_columns ) select_all_columns = ", ".join(escape_sqlite(t) for t in table_columns) + # rowid tables (no specified primary key) need a different SELECT use_rowid = not pks and not is_view if use_rowid: select_specified_columns = f"rowid, {select_specified_columns}" @@ -487,7 +492,7 @@ class TableView(RowTableShared): f'{through_table}.{other_column} = "{value}"' ) - # _search support: + # _search= support: fts_table = special_args.get("_fts_table") fts_table = fts_table or table_metadata.get("fts_table") fts_table = fts_table or await db.fts_table(table) @@ -541,8 +546,6 @@ class TableView(RowTableShared): ) params[f"search_{i}"] = search_text - sortable_columns = set() - sortable_columns = await self.sortable_columns_for_table( database, table, use_rowid ) @@ -581,6 +584,7 @@ class TableView(RowTableShared): count_sql = f"select count(*) {from_sql}" + # Handl pagination driven by ?_next= _next = _next or special_args.get("_next") offset = "" if _next: @@ -679,6 +683,7 @@ class TableView(RowTableShared): else: page_size = self.ds.page_size + # Facets are calculated against SQL without order by or limit sql_no_order_no_limit = ( "select {select_all_columns} from {table_name} {where}".format( select_all_columns=select_all_columns, @@ -686,6 +691,8 @@ class TableView(RowTableShared): where=where_clause, ) ) + + # This is the SQL that populates the main table on the page sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( select_specified_columns=select_specified_columns, table_name=escape_sqlite(table), @@ -698,15 +705,17 @@ class TableView(RowTableShared): if request.args.get("_timelimit"): extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) + # Execute the main query! results = await db.execute(sql, params, truncate=True, **extra_args) - # Number of filtered rows in whole set: + # Calculate the total count for this query filtered_table_rows_count = None if ( not db.is_mutable and self.ds.inspect_data and count_sql == f"select count(*) from {table} " ): + # We can use a previously cached table row count try: filtered_table_rows_count = self.ds.inspect_data[database]["tables"][ table @@ -714,6 +723,7 @@ class TableView(RowTableShared): except KeyError: pass + # Otherwise run a select count(*) ... if count_sql and filtered_table_rows_count is None and not nocount: try: count_rows = list(await db.execute(count_sql, from_sql_params)) @@ -721,7 +731,7 @@ class TableView(RowTableShared): except QueryInterrupted: pass - # facets support + # Faceting if not self.ds.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): @@ -764,6 +774,18 @@ class TableView(RowTableShared): facet_results[key] = facet_info facets_timed_out.extend(instance_facets_timed_out) + # Calculate suggested facets + suggested_facets = [] + if ( + self.ds.setting("suggest_facets") + and self.ds.setting("allow_facet") + and not _next + and not nofacet + and not nosuggest + ): + for facet in facet_instances: + suggested_facets.extend(await facet.suggest()) + # Figure out columns and rows for the query columns = [r[0] for r in results.description] rows = list(results.rows) @@ -846,19 +868,6 @@ class TableView(RowTableShared): ) rows = rows[:page_size] - # Detect suggested facets - suggested_facets = [] - - if ( - self.ds.setting("suggest_facets") - and self.ds.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - for facet in facet_instances: - suggested_facets.extend(await facet.suggest()) - # human_description_en combines filters AND search, if provided human_description_en = filters.human_description_en( extra=extra_human_descriptions From 0663d5525cc41e9260ac7d1f6386d3a6eb5ad2a9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Dec 2021 14:00:29 -0800 Subject: [PATCH 0537/1866] More comments in TableView.data(), refs #1518 --- datasette/views/table.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d37a3066..da263966 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -437,6 +437,8 @@ class TableView(RowTableShared): table_metadata = self.ds.table_metadata(database, table) units = table_metadata.get("units", {}) + + # Build where clauses from query string arguments filters = Filters(sorted(other_args), units, ureg) where_clauses, params = filters.build_where_clauses(table) @@ -584,7 +586,7 @@ class TableView(RowTableShared): count_sql = f"select count(*) {from_sql}" - # Handl pagination driven by ?_next= + # Handle pagination driven by ?_next= _next = _next or special_args.get("_next") offset = "" if _next: From aa7f0037a46eb76ae6fe9bf2a1f616c58738ecdf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Dec 2021 11:02:14 -0800 Subject: [PATCH 0538/1866] filters_from_request plugin hook, now used in TableView - New `filters_from_request` plugin hook, closes #473 - Used it to extract the logic from TableView that handles `_search` and `_through` and `_where` - refs #1518 Also needed for this plugin work: https://github.com/simonw/datasette-leaflet-freedraw/issues/7 --- datasette/filters.py | 167 ++++++++++++++++++++++++++++++++++++++- datasette/hookspecs.py | 11 +++ datasette/plugins.py | 1 + datasette/views/table.py | 127 +++++------------------------ docs/plugin_hooks.rst | 53 +++++++++++++ tests/test_filters.py | 87 +++++++++++++++++++- tests/test_plugins.py | 18 +++++ 7 files changed, 353 insertions(+), 111 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index cbd94415..5ea3488b 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,7 +1,172 @@ +from datasette import hookimpl +from datasette.views.base import DatasetteError +from datasette.utils.asgi import BadRequest import json import numbers +from .utils import detect_json1, escape_sqlite, path_with_removed_args -from .utils import detect_json1, escape_sqlite + +@hookimpl(specname="filters_from_request") +def where_filters(request, database, datasette): + # This one deals with ?_where= + async def inner(): + where_clauses = [] + extra_wheres_for_ui = [] + if "_where" in request.args: + if not await datasette.permission_allowed( + request.actor, + "execute-sql", + resource=database, + default=True, + ): + raise DatasetteError("_where= is not allowed", status=403) + else: + where_clauses.extend(request.args.getlist("_where")) + extra_wheres_for_ui = [ + { + "text": text, + "remove_url": path_with_removed_args(request, {"_where": text}), + } + for text in request.args.getlist("_where") + ] + + return FilterArguments( + where_clauses, + extra_context={ + "extra_wheres_for_ui": extra_wheres_for_ui, + }, + ) + + return inner + + +@hookimpl(specname="filters_from_request") +def search_filters(request, database, table, datasette): + # ?_search= and _search_colname= + async def inner(): + where_clauses = [] + params = {} + human_descriptions = [] + extra_context = {} + + # Figure out which fts_table to use + table_metadata = datasette.table_metadata(database, table) + db = datasette.get_database(database) + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") + fts_table = fts_table or await db.fts_table(table) + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + search_args = { + key: request.args[key] + for key in request.args + if key.startswith("_search") and key != "_searchmode" + } + search = "" + search_mode_raw = table_metadata.get("searchmode") == "raw" + # Or set search mode from the querystring + qs_searchmode = request.args.get("_searchmode") + if qs_searchmode == "escaped": + search_mode_raw = False + if qs_searchmode == "raw": + search_mode_raw = True + + extra_context["supports_search"] = bool(fts_table) + + if fts_table and search_args: + if "_search" in search_args: + # Simple ?_search=xxx + search = search_args["_search"] + where_clauses.append( + "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( + fts_table=escape_sqlite(fts_table), + fts_pk=escape_sqlite(fts_pk), + match_clause=":search" + if search_mode_raw + else "escape_fts(:search)", + ) + ) + human_descriptions.append(f'search matches "{search}"') + params["search"] = search + extra_context["search"] = search + else: + # More complex: search against specific columns + for i, (key, search_text) in enumerate(search_args.items()): + search_col = key.split("_search_", 1)[1] + if search_col not in await db.table_columns(fts_table): + raise BadRequest("Cannot search by that column") + + where_clauses.append( + "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( + fts_table=escape_sqlite(fts_table), + search_col=escape_sqlite(search_col), + match_clause=":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i), + ) + ) + human_descriptions.append( + f'search column "{search_col}" matches "{search_text}"' + ) + params[f"search_{i}"] = search_text + extra_context["search"] = search_text + + return FilterArguments(where_clauses, params, human_descriptions, extra_context) + + return inner + + +@hookimpl(specname="filters_from_request") +def through_filters(request, database, table, datasette): + # ?_search= and _search_colname= + async def inner(): + where_clauses = [] + params = {} + human_descriptions = [] + extra_context = {} + + # Support for ?_through={table, column, value} + if "_through" in request.args: + for through in request.args.getlist("_through"): + through_data = json.loads(through) + through_table = through_data["table"] + other_column = through_data["column"] + value = through_data["value"] + db = datasette.get_database(database) + outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) + try: + fk_to_us = [ + fk for fk in outgoing_foreign_keys if fk["other_table"] == table + ][0] + except IndexError: + raise DatasetteError( + "Invalid _through - could not find corresponding foreign key" + ) + param = f"p{len(params)}" + where_clauses.append( + "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format( + through_table=escape_sqlite(through_table), + our_pk=escape_sqlite(fk_to_us["other_column"]), + our_column=escape_sqlite(fk_to_us["column"]), + other_column=escape_sqlite(other_column), + param=param, + ) + ) + params[param] = value + human_descriptions.append(f'{through_table}.{other_column} = "{value}"') + + return FilterArguments(where_clauses, params, human_descriptions, extra_context) + + return inner + + +class FilterArguments: + def __init__( + self, where_clauses, params=None, human_descriptions=None, extra_context=None + ): + self.where_clauses = where_clauses + self.params = params or {} + self.human_descriptions = human_descriptions or [] + self.extra_context = extra_context or {} class Filter: diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 1d4e3b27..8f4fecab 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -89,6 +89,17 @@ def actor_from_request(datasette, request): """Return an actor dictionary based on the incoming request""" +@hookspec +def filters_from_request(request, database, table, datasette): + """ + Return datasette.filters.FilterArguments( + where_clauses=[str, str, str], + params={}, + human_descriptions=[str, str, str], + extra_context={} + ) based on the request""" + + @hookspec def permission_allowed(datasette, actor, action, resource): """Check if actor is allowed to perform this action - return True, False or None""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 50791988..76b46a47 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -8,6 +8,7 @@ DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", "datasette.facets", + "datasette.filters", "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", diff --git a/datasette/views/table.py b/datasette/views/table.py index da263966..cfd31bd3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -442,117 +442,27 @@ class TableView(RowTableShared): filters = Filters(sorted(other_args), units, ureg) where_clauses, params = filters.build_where_clauses(table) - extra_wheres_for_ui = [] - # Add _where= from querystring - if "_where" in request.args: - if not await self.ds.permission_allowed( - request.actor, - "execute-sql", - resource=database, - default=True, - ): - raise DatasetteError("_where= is not allowed", status=403) - else: - where_clauses.extend(request.args.getlist("_where")) - extra_wheres_for_ui = [ - { - "text": text, - "remove_url": path_with_removed_args(request, {"_where": text}), - } - for text in request.args.getlist("_where") - ] - - # Support for ?_through={table, column, value} + # Execute filters_from_request plugin hooks + extra_context_from_filters = {} extra_human_descriptions = [] - if "_through" in request.args: - for through in request.args.getlist("_through"): - through_data = json.loads(through) - through_table = through_data["table"] - other_column = through_data["column"] - value = through_data["value"] - outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - try: - fk_to_us = [ - fk for fk in outgoing_foreign_keys if fk["other_table"] == table - ][0] - except IndexError: - raise DatasetteError( - "Invalid _through - could not find corresponding foreign key" - ) - param = f"p{len(params)}" - where_clauses.append( - "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format( - through_table=escape_sqlite(through_table), - our_pk=escape_sqlite(fk_to_us["other_column"]), - our_column=escape_sqlite(fk_to_us["column"]), - other_column=escape_sqlite(other_column), - param=param, - ) - ) - params[param] = value - extra_human_descriptions.append( - f'{through_table}.{other_column} = "{value}"' - ) - # _search= support: - fts_table = special_args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") - fts_table = fts_table or await db.fts_table(table) - fts_pk = special_args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) - search_args = dict( - pair - for pair in special_args.items() - if pair[0].startswith("_search") and pair[0] != "_searchmode" - ) - search = "" - search_mode_raw = table_metadata.get("searchmode") == "raw" - # Or set it from the querystring - qs_searchmode = special_args.get("_searchmode") - if qs_searchmode == "escaped": - search_mode_raw = False - if qs_searchmode == "raw": - search_mode_raw = True - if fts_table and search_args: - if "_search" in search_args: - # Simple ?_search=xxx - search = search_args["_search"] - where_clauses.append( - "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( - fts_table=escape_sqlite(fts_table), - fts_pk=escape_sqlite(fts_pk), - match_clause=":search" - if search_mode_raw - else "escape_fts(:search)", - ) - ) - extra_human_descriptions.append(f'search matches "{search}"') - params["search"] = search - else: - # More complex: search against specific columns - for i, (key, search_text) in enumerate(search_args.items()): - search_col = key.split("_search_", 1)[1] - if search_col not in await db.table_columns(fts_table): - raise BadRequest("Cannot search by that column") - - where_clauses.append( - "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( - fts_table=escape_sqlite(fts_table), - search_col=escape_sqlite(search_col), - match_clause=":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i), - ) - ) - extra_human_descriptions.append( - f'search column "{search_col}" matches "{search_text}"' - ) - params[f"search_{i}"] = search_text + for hook in pm.hook.filters_from_request( + request=request, + table=table, + database=database, + datasette=self.ds, + ): + filter_arguments = await await_me_maybe(hook) + if filter_arguments: + where_clauses.extend(filter_arguments.where_clauses) + params.update(filter_arguments.params) + extra_human_descriptions.extend(filter_arguments.human_descriptions) + extra_context_from_filters.update(filter_arguments.extra_context) + # Deal with custom sort orders sortable_columns = await self.sortable_columns_for_table( database, table, use_rowid ) - - # Allow for custom sort order sort = special_args.get("_sort") sort_desc = special_args.get("_sort_desc") @@ -942,10 +852,8 @@ class TableView(RowTableShared): for table_column in table_columns if table_column not in columns ] - return { + d = { "table_actions": table_actions, - "supports_search": bool(fts_table), - "search": search or "", "use_rowid": use_rowid, "filters": filters, "display_columns": display_columns, @@ -957,7 +865,6 @@ class TableView(RowTableShared): key=lambda f: (len(f["results"]), f["name"]), reverse=True, ), - "extra_wheres_for_ui": extra_wheres_for_ui, "form_hidden_args": form_hidden_args, "is_sortable": any(c["sortable"] for c in display_columns), "fix_path": self.ds.urls.path, @@ -977,6 +884,8 @@ class TableView(RowTableShared): "view_definition": await db.get_view_definition(table), "table_definition": await db.get_table_definition(table), } + d.update(extra_context_from_filters) + return d return ( { diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4a7c36c3..d76f70e5 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -923,6 +923,59 @@ Instead of returning a dictionary, this function can return an awaitable functio Example: `datasette-auth-tokens `_ +.. _plugin_hook_filters_from_request: + +filters_from_request(request, database, table, datasette) +--------------------------------------------------------- + +``request`` - object + The current HTTP :ref:`internals_request`. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +This hook runs on the :ref:`table ` page, and can influence the ``where`` clause of the SQL query used to populate that page, based on query string arguments on the incoming request. + +The hook should return an instance of ``datasette.filters.FilterArguments`` which has one required and three optional arguments: + +.. code-block:: python + + return FilterArguments( + where_clauses=["id > :max_id"], + params={"max_id": 5}, + human_descriptions=["max_id is greater than 5"], + extra_context={} + ) + +The arguments to the ``FilterArguments`` class constructor are as follows: + +``where_clauses`` - list of strings, required + A list of SQL fragments that will be inserted into the SQL query, joined by the ``and`` operator. These can include ``:named`` parameters which will be populated using data in ``params``. +``params`` - dictionary, optional + Additional keyword arguments to be used when the query is executed. These should match any ``:arguments`` in the where clauses. +``human_descriptions`` - list of strings, optional + These strings will be included in the human-readable description at the top of the page and the page ````. +``extra_context`` - dictionary, optional + Additional context variables that should be made available to the ``table.html`` template when it is rendered. + +This example plugin causes 0 results to be returned if ``?_nothing=1`` is added to the URL: + +.. code-block:: python + + from datasette import hookimpl + from datasette.filters import FilterArguments + + @hookimpl + def filters_from_request(self, request): + if request.args.get("_nothing"): + return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) + .. _plugin_hook_permission_allowed: permission_allowed(datasette, actor, action, resource) diff --git a/tests/test_filters.py b/tests/test_filters.py index d05ae80f..2ff57489 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,4 +1,6 @@ -from datasette.filters import Filters +from datasette.filters import Filters, through_filters, where_filters, search_filters +from datasette.utils.asgi import Request +from .fixtures import app_client import pytest @@ -74,3 +76,86 @@ def test_build_where(args, expected_where, expected_params): sql_bits, actual_params = f.build_where_clauses("table") assert expected_where == sql_bits assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params + + +@pytest.mark.asyncio +async def test_through_filters_from_request(app_client): + request = Request.fake( + '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + ) + filter_args = await ( + through_filters( + request=request, + datasette=app_client.ds, + table="roadside_attractions", + database="fixtures", + ) + )() + assert filter_args.where_clauses == [ + "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)" + ] + assert filter_args.params == {"p0": "1"} + assert filter_args.human_descriptions == [ + 'roadside_attraction_characteristics.characteristic_id = "1"' + ] + assert filter_args.extra_context == {} + + +@pytest.mark.asyncio +async def test_through_filters_from_request(app_client): + request = Request.fake( + '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + ) + filter_args = await ( + through_filters( + request=request, + datasette=app_client.ds, + table="roadside_attractions", + database="fixtures", + ) + )() + assert filter_args.where_clauses == [ + "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)" + ] + assert filter_args.params == {"p0": "1"} + assert filter_args.human_descriptions == [ + 'roadside_attraction_characteristics.characteristic_id = "1"' + ] + assert filter_args.extra_context == {} + + +@pytest.mark.asyncio +async def test_where_filters_from_request(app_client): + request = Request.fake("/?_where=pk+>+3") + filter_args = await ( + where_filters( + request=request, + datasette=app_client.ds, + database="fixtures", + ) + )() + assert filter_args.where_clauses == ["pk > 3"] + assert filter_args.params == {} + assert filter_args.human_descriptions == [] + assert filter_args.extra_context == { + "extra_wheres_for_ui": [{"text": "pk > 3", "remove_url": "/"}] + } + + +@pytest.mark.asyncio +async def test_search_filters_from_request(app_client): + request = Request.fake("/?_search=bobcat") + filter_args = await ( + search_filters( + request=request, + datasette=app_client.ds, + database="fixtures", + table="searchable", + ) + )() + assert filter_args.where_clauses == [ + "rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))" + ] + assert filter_args.params == {"search": "bobcat"} + assert filter_args.human_descriptions == ['search matches "bobcat"'] + assert filter_args.extra_context == {"supports_search": True, "search": "bobcat"} diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1da28453..656f39e4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,6 +9,7 @@ from .fixtures import ( from click.testing import CliRunner from datasette.app import Datasette from datasette import cli, hookimpl +from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 from datasette.utils import CustomRow @@ -977,3 +978,20 @@ def test_hook_register_commands(): } pm.unregister(name="verify") importlib.reload(cli) + + +def test_hook_filters_from_request(app_client): + class ReturnNothingPlugin: + __name__ = "ReturnNothingPlugin" + + @hookimpl + def filters_from_request(self, request): + if request.args.get("_nothing"): + return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) + + pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin") + response = app_client.get("/fixtures/facetable?_nothing=1") + assert "0 rows\n where NOTHING" in response.text + json_response = app_client.get("/fixtures/facetable.json?_nothing=1") + assert json_response.json["rows"] == [] + pm.unregister(name="ReturnNothingPlugin") From 92a5280d2e75c39424a75ad6226fc74400ae984f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 11:13:02 -0800 Subject: [PATCH 0539/1866] Release 0.60a0 Refs #473, #625, #1544, #1551, #1552, #1556, #1557 --- datasette/version.py | 2 +- docs/changelog.rst | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 9c85b763..2fce006c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.59.4" +__version__ = "0.60a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ddc2794..92a9d941 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,19 @@ Changelog ========= +.. _v0_60a0: + +0.60a0 (2021-12-17) +------------------- + +- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`) +- The number of unique values in a facet is now always displayed. Previously it was only displayed if the user specified ``?_facet_size=max``. (:issue:`1556`) +- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`) +- Facets of type ``date`` or ``array`` can now be configured in ``metadata.json``, see :ref:`facets_metadata`. Thanks, David Larlet. (:issue:`1552`) +- New ``?_nosuggest=1`` parameter for table views, which disables facet suggestion. (:issue:`1557`) +- Label columns detected for foreign keys are now case-insensitive, so ``Name`` or ``TITLE`` will be detected in the same way as ``name`` or ``title``. (:issue:`1544`) +- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`) + .. _v0_59_4: 0.59.4 (2021-11-29) From f000a7bd75ac512478070f2e2a09c8fb9604c82d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 12:15:29 -0800 Subject: [PATCH 0540/1866] Use load_extension(?) instead of fstring --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 28268e42..715506bd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -553,7 +553,7 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - conn.execute(f"SELECT load_extension('{extension}')") + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member From 35cba9e85a574cebf2986b64107fa84d02bd86ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 15:08:28 -0800 Subject: [PATCH 0541/1866] Update janus requirement from <0.8,>=0.6.2 to >=0.6.2,<1.1 (#1562) Updates the requirements on [janus](https://github.com/aio-libs/janus) to permit the latest version. - [Release notes](https://github.com/aio-libs/janus/releases) - [Changelog](https://github.com/aio-libs/janus/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/janus/compare/v0.6.2...v1.0.0) --- updated-dependencies: - dependency-name: janus dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 534265c2..f8cd3e5b 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ setup( "pluggy>=0.13,<1.1", "uvicorn~=0.11", "aiofiles>=0.4,<0.9", - "janus>=0.6.2,<0.8", + "janus>=0.6.2,<1.1", "asgi-csrf>=0.9", "PyYAML>=5.3,<7.0", "mergedeep>=1.1.1,<1.4.0", From d0f24f9bbc596873f261ed4e0267c4aa5a0bac2b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 15:28:26 -0800 Subject: [PATCH 0542/1866] Clarifying comment The new filters stuff is a little bit action-at-a-distance --- datasette/views/table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index cfd31bd3..c3bcf01d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -442,7 +442,8 @@ class TableView(RowTableShared): filters = Filters(sorted(other_args), units, ureg) where_clauses, params = filters.build_where_clauses(table) - # Execute filters_from_request plugin hooks + # Execute filters_from_request plugin hooks - including the default + # ones that live in datasette/filters.py extra_context_from_filters = {} extra_human_descriptions = [] From 0c91e59d2bbfc08884cfcf5d1b902a2f4968b7ff Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 15:28:44 -0800 Subject: [PATCH 0543/1866] datasette-leaflet-freedraw is an example of filters_from_request --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d76f70e5..cbaf4c54 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -976,6 +976,8 @@ This example plugin causes 0 results to be returned if ``?_nothing=1`` is added if request.args.get("_nothing"): return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) +Example: `datasette-leaflet-freedraw <https://datasette.io/plugins/datasette-leaflet-freedraw>`_ + .. _plugin_hook_permission_allowed: permission_allowed(datasette, actor, action, resource) From c35b84a2aabe2f14aeacf6cda4110ae1e94d6059 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 17:54:39 -0800 Subject: [PATCH 0544/1866] Remove undocumented sqlite_functions mechanism, closes #1567 --- datasette/app.py | 3 --- tests/fixtures.py | 1 - tests/plugins/sleep_sql_function.py | 7 +++++++ 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 tests/plugins/sleep_sql_function.py diff --git a/datasette/app.py b/datasette/app.py index 715506bd..d94cd5a2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -259,7 +259,6 @@ class Datasette: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) self._metadata_local = metadata or {} - self.sqlite_functions = [] self.sqlite_extensions = [] for extension in sqlite_extensions or []: # Resolve spatialite, if requested @@ -548,8 +547,6 @@ class Datasette: def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") - for name, num_args, func in self.sqlite_functions: - conn.create_function(name, num_args, func) if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: diff --git a/tests/fixtures.py b/tests/fixtures.py index 37399da0..76f794c6 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -153,7 +153,6 @@ def make_app_client( template_dir=template_dir, crossdb=crossdb, ) - ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) yield TestClient(ds) diff --git a/tests/plugins/sleep_sql_function.py b/tests/plugins/sleep_sql_function.py new file mode 100644 index 00000000..d4b32a09 --- /dev/null +++ b/tests/plugins/sleep_sql_function.py @@ -0,0 +1,7 @@ +from datasette import hookimpl +import time + + +@hookimpl +def prepare_connection(conn): + conn.create_function("sleep", 1, lambda n: time.sleep(float(n))) From 83bacfa9452babe7bd66e3579e23af988d00f6ac Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 17:58:39 -0800 Subject: [PATCH 0545/1866] Call _prepare_connection() on write connections, closes #1564 --- datasette/database.py | 1 + tests/test_internals_database.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/datasette/database.py b/datasette/database.py index d1217e18..0a0c104a 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -128,6 +128,7 @@ class Database: conn = None try: conn = self.connect(write=True) + self.ds._prepare_connection(conn, self.name) except Exception as e: conn_exception = e while True: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index a00fe447..609caabf 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -396,6 +396,12 @@ async def test_execute_write_block_false(db): assert "Mystery!" == rows.rows[0][0] +@pytest.mark.asyncio +async def test_execute_write_has_correctly_prepared_connection(db): + # The sleep() function is only available if ds._prepare_connection() was called + await db.execute_write("select sleep(0.01)", block=True) + + @pytest.mark.asyncio async def test_execute_write_fn_block_false(db): def write_fn(conn): From 359140cedaf69242d6356479fb8a9d3aa591e618 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 18:09:00 -0800 Subject: [PATCH 0546/1866] Datasette() constructor no longer requires files=, closes #1563 --- datasette/app.py | 4 ++-- tests/test_internals_datasette.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d94cd5a2..17fa06a5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -191,7 +191,7 @@ class Datasette: def __init__( self, - files, + files=None, immutables=None, cache_headers=True, cors=False, @@ -214,7 +214,7 @@ class Datasette: ), "config_dir= should be a pathlib.Path" self.pdb = pdb self._secret = secret or secrets.token_hex(32) - self.files = tuple(files) + tuple(immutables or []) + self.files = tuple(files or []) + tuple(immutables or []) if config_dir: self.files += tuple([str(p) for p in config_dir.glob("*.db")]) if ( diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 56bc2fb4..692312a7 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,6 +1,7 @@ """ Tests for the datasette.app.Datasette class """ +from datasette.app import Datasette from itsdangerous import BadSignature from .fixtures import app_client import pytest @@ -45,3 +46,19 @@ def test_sign_unsign(datasette, value, namespace): ) def test_datasette_setting(datasette, setting, expected): assert datasette.setting(setting) == expected + + +@pytest.mark.asyncio +async def test_datasette_constructor(): + ds = Datasette(memory=True) + databases = (await ds.client.get("/-/databases.json")).json() + assert databases == [ + { + "name": "_memory", + "path": None, + "size": 0, + "is_mutable": False, + "is_memory": True, + "hash": None, + } + ] From 3a0cae4d7f77b5c2a103ea74ca7fa7a0d9ff2e66 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 18:19:09 -0800 Subject: [PATCH 0547/1866] Fix bug introduced by refactor in c35b84a2aabe2f14aeacf6cda4110ae1e94d6059 --- tests/fixtures.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/fixtures.py b/tests/fixtures.py index 76f794c6..26f0cf7b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -83,6 +83,13 @@ EXPECTED_PLUGINS = [ "version": None, "hooks": ["register_output_renderer"], }, + { + "name": "sleep_sql_function.py", + "static": False, + "templates": False, + "version": None, + "hooks": ["prepare_connection"], + }, { "name": "view_name.py", "static": False, From 7c8f8aa209e4ba7bf83976f8495d67c28fbfca24 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 18:19:36 -0800 Subject: [PATCH 0548/1866] Documentation for Datasette() constructor, closes #1563 --- docs/internals.rst | 21 +++++++++++++++++++++ tests/test_internals_datasette.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 411327eb..c706031b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -196,6 +196,27 @@ Datasette class This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``. +You can create your own instance of this - for example to help write tests for a plugin - like so: + +.. code-block:: python + + from datasette.app import Datasette + + # With no arguments a single in-memory database will be attached + datasette = Datasette() + + # The files= argument can load files from disk + datasette = Datasette(files="/path/to/my-database.db") + + # Pass metadata as a JSON dictionary like this + datasette = Datasette(files="/path/to/my-database.db", metadata={ + "databases": { + "my-database": { + "description": "This is my database" + } + } + }) + .. _datasette_databases: .databases diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 692312a7..adf84be9 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -50,7 +50,7 @@ def test_datasette_setting(datasette, setting, expected): @pytest.mark.asyncio async def test_datasette_constructor(): - ds = Datasette(memory=True) + ds = Datasette() databases = (await ds.client.get("/-/databases.json")).json() assert databases == [ { From f81d9d0cd9f567e73a1a54be34b653db8ae2c1cf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Dec 2021 18:42:29 -0800 Subject: [PATCH 0549/1866] Trace write SQL queries in addition to read ones, closes #1568 --- datasette/database.py | 4 +++- tests/test_api.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index 0a0c104a..468e9360 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -99,7 +99,9 @@ class Database: with conn: return conn.execute(sql, params or []) - return await self.execute_write_fn(_inner, block=block) + with trace("sql", database=self.name, sql=sql.strip(), params=params): + results = await self.execute_write_fn(_inner, block=block) + return results async def execute_write_fn(self, fn, block=False): task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") diff --git a/tests/test_api.py b/tests/test_api.py index df9e0fc4..9ad7d569 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -939,6 +939,19 @@ def test_trace(trace_debug): assert isinstance(trace["sql"], str) assert isinstance(trace["params"], (list, dict, None.__class__)) + sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace] + # There should be a mix of different types of SQL statement + expected = ( + "CREATE TABLE ", + "PRAGMA ", + "INSERT OR REPLACE INTO ", + "DELETE FROM ", + "INSERT INTO", + "select ", + ) + for prefix in expected: + assert any(sql.startswith(prefix) for sql in sqls) + @pytest.mark.parametrize( "path,status_code", From 85c22f4fbccb7b35fbc16d3ef035ca71b1a5a20a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 10:10:37 -0800 Subject: [PATCH 0550/1866] Corrected Datasette(files=) example from #1563 --- docs/internals.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index c706031b..8788b26a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -206,10 +206,10 @@ You can create your own instance of this - for example to help write tests for a datasette = Datasette() # The files= argument can load files from disk - datasette = Datasette(files="/path/to/my-database.db") + datasette = Datasette(files=["/path/to/my-database.db"]) # Pass metadata as a JSON dictionary like this - datasette = Datasette(files="/path/to/my-database.db", metadata={ + datasette = Datasette(files=["/path/to/my-database.db"], metadata={ "databases": { "my-database": { "description": "This is my database" From 9e094b7c9d575320a2f0c956eb547bfcf6d64d39 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 10:28:25 -0800 Subject: [PATCH 0551/1866] db.execute_write(executescript=True) option, closes #1569 --- datasette/database.py | 8 ++++++-- docs/internals.rst | 6 ++++-- tests/test_internals_database.py | 21 +++++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 468e9360..350c4e9c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -94,10 +94,14 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False ) - async def execute_write(self, sql, params=None, block=False): + async def execute_write(self, sql, params=None, executescript=False, block=False): + assert not (executescript and params), "Cannot use params with executescript=True" def _inner(conn): with conn: - return conn.execute(sql, params or []) + if executescript: + return conn.executescript(sql) + else: + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block) diff --git a/docs/internals.rst b/docs/internals.rst index 8788b26a..d40e679b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -663,8 +663,8 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, block=False) ------------------------------------------------------ +await db.execute_write(sql, params=None, executescript=False, block=False) +-------------------------------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -676,6 +676,8 @@ By default queries are considered to be "fire and forget" - they will be added t If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. +If you pass ``executescript=True`` your SQL will be executed using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. This allows multiple SQL statements to be separated by semicolons, but cannot be used with the ``params=`` option. + .. _database_execute_write_fn: await db.execute_write_fn(fn, block=False) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 609caabf..0a5c01a3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -396,6 +396,27 @@ async def test_execute_write_block_false(db): assert "Mystery!" == rows.rows[0][0] +@pytest.mark.asyncio +async def test_execute_write_executescript(db): + await db.execute_write( + "create table foo (id integer primary key); create table bar (id integer primary key); ", + executescript=True, + block=True + ) + table_names = await db.table_names() + assert {"foo", "bar"}.issubset(table_names) + + +@pytest.mark.asyncio +async def test_execute_write_executescript_not_allowed_with_params(db): + with pytest.raises(AssertionError): + await db.execute_write( + "update roadside_attractions set name = ? where pk = ?", + ["Mystery!", 1], + executescript=True + ) + + @pytest.mark.asyncio async def test_execute_write_has_correctly_prepared_connection(db): # The sleep() function is only available if ds._prepare_connection() was called From 2e4ba71b53a45a7d2273afd30e400002c7f39755 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 10:30:53 -0800 Subject: [PATCH 0552/1866] Optimize create table calls using executescript=True Refs #1555, #1569 --- datasette/database.py | 5 +++- datasette/utils/internal_db.py | 48 ++++++-------------------------- tests/test_api.py | 4 ++- tests/test_internals_database.py | 4 +-- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 350c4e9c..f8365f5c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -95,7 +95,10 @@ class Database: ) async def execute_write(self, sql, params=None, executescript=False, block=False): - assert not (executescript and params), "Cannot use params with executescript=True" + assert not ( + executescript and params + ), "Cannot use params with executescript=True" + def _inner(conn): with conn: if executescript: diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 40fe719e..80babff8 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -2,22 +2,14 @@ import textwrap async def init_internal_db(db): - await db.execute_write( - textwrap.dedent( - """ + create_tables_sql = textwrap.dedent( + """ CREATE TABLE IF NOT EXISTS databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER - ) - """ - ), - block=True, - ) - await db.execute_write( - textwrap.dedent( - """ + ); CREATE TABLE IF NOT EXISTS tables ( database_name TEXT, table_name TEXT, @@ -25,14 +17,7 @@ async def init_internal_db(db): sql TEXT, PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) - ) - """ - ), - block=True, - ) - await db.execute_write( - textwrap.dedent( - """ + ); CREATE TABLE IF NOT EXISTS columns ( database_name TEXT, table_name TEXT, @@ -46,14 +31,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) - ) - """ - ), - block=True, - ) - await db.execute_write( - textwrap.dedent( - """ + ); CREATE TABLE IF NOT EXISTS indexes ( database_name TEXT, table_name TEXT, @@ -65,14 +43,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) - ) - """ - ), - block=True, - ) - await db.execute_write( - textwrap.dedent( - """ + ); CREATE TABLE IF NOT EXISTS foreign_keys ( database_name TEXT, table_name TEXT, @@ -87,11 +58,10 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name, id, seq), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) - ) + ); """ - ), - block=True, - ) + ).strip() + await db.execute_write(create_tables_sql, block=True, executescript=True) async def populate_schema_tables(internal_db, db): diff --git a/tests/test_api.py b/tests/test_api.py index 9ad7d569..29c92920 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -950,7 +950,9 @@ def test_trace(trace_debug): "select ", ) for prefix in expected: - assert any(sql.startswith(prefix) for sql in sqls) + assert any( + sql.startswith(prefix) for sql in sqls + ), "No trace beginning with: {}".format(prefix) @pytest.mark.parametrize( diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0a5c01a3..aa5676e7 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -401,7 +401,7 @@ async def test_execute_write_executescript(db): await db.execute_write( "create table foo (id integer primary key); create table bar (id integer primary key); ", executescript=True, - block=True + block=True, ) table_names = await db.table_names() assert {"foo", "bar"}.issubset(table_names) @@ -413,7 +413,7 @@ async def test_execute_write_executescript_not_allowed_with_params(db): await db.execute_write( "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1], - executescript=True + executescript=True, ) From 5cadc244895fc47e0534c6e90df976d34293921e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 10:57:22 -0800 Subject: [PATCH 0553/1866] db.execute_write_script() and db.execute_write_many(), closes #1570 Refs #1555 --- datasette/database.py | 29 ++++++++++++++++++++--------- datasette/utils/internal_db.py | 2 +- docs/internals.rst | 26 +++++++++++++++++++++++--- tests/test_internals_database.py | 24 ++++++++++++++---------- 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index f8365f5c..1de1d5ec 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -94,22 +94,33 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False ) - async def execute_write(self, sql, params=None, executescript=False, block=False): - assert not ( - executescript and params - ), "Cannot use params with executescript=True" - + async def execute_write(self, sql, params=None, block=False): def _inner(conn): with conn: - if executescript: - return conn.executescript(sql) - else: - return conn.execute(sql, params or []) + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block) return results + async def execute_write_script(self, sql, block=False): + def _inner(conn): + with conn: + return conn.executescript(sql) + + with trace("sql", database=self.name, sql=sql.strip(), executescript=True): + results = await self.execute_write_fn(_inner, block=block) + return results + + async def execute_write_many(self, sql, params_seq, block=False): + def _inner(conn): + with conn: + return conn.executemany(sql, params_seq) + + with trace("sql", database=self.name, sql=sql.strip(), executemany=True): + results = await self.execute_write_fn(_inner, block=block) + return results + async def execute_write_fn(self, fn, block=False): task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") if self._write_queue is None: diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 80babff8..8a145767 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -61,7 +61,7 @@ async def init_internal_db(db): ); """ ).strip() - await db.execute_write(create_tables_sql, block=True, executescript=True) + await db.execute_write_script(create_tables_sql, block=True) async def populate_schema_tables(internal_db, db): diff --git a/docs/internals.rst b/docs/internals.rst index d40e679b..bc0174a8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -663,8 +663,8 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, executescript=False, block=False) --------------------------------------------------------------------------- +await db.execute_write(sql, params=None, block=False) +----------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -676,7 +676,27 @@ By default queries are considered to be "fire and forget" - they will be added t If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. -If you pass ``executescript=True`` your SQL will be executed using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. This allows multiple SQL statements to be separated by semicolons, but cannot be used with the ``params=`` option. +.. _database_execute_write_script: + +await db.execute_write_script(sql, block=False) +----------------------------------------------- + +Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. + +.. _database_execute_write_many: + +await db.execute_write_many(sql, params_seq, block=False) +--------------------------------------------------------- + +Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executemany>`__ method. This will efficiently execute the same SQL statement against each of the parameters in the ``params_seq`` iterator, for example: + +.. code-block:: python + + await db.execute_write_many( + "insert into characters (id, name) values (?, ?)", + [(1, "Melanie"), (2, "Selma"), (2, "Viktor")], + block=True, + ) .. _database_execute_write_fn: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index aa5676e7..f751bf9a 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -397,10 +397,9 @@ async def test_execute_write_block_false(db): @pytest.mark.asyncio -async def test_execute_write_executescript(db): - await db.execute_write( +async def test_execute_write_script(db): + await db.execute_write_script( "create table foo (id integer primary key); create table bar (id integer primary key); ", - executescript=True, block=True, ) table_names = await db.table_names() @@ -408,13 +407,18 @@ async def test_execute_write_executescript(db): @pytest.mark.asyncio -async def test_execute_write_executescript_not_allowed_with_params(db): - with pytest.raises(AssertionError): - await db.execute_write( - "update roadside_attractions set name = ? where pk = ?", - ["Mystery!", 1], - executescript=True, - ) +async def test_execute_write_many(db): + await db.execute_write_script( + "create table foomany (id integer primary key)", + block=True, + ) + await db.execute_write_many( + "insert into foomany (id) values (?)", + [(1,), (10,), (100,)], + block=True, + ) + result = await db.execute("select * from foomany") + assert [r[0] for r in result.rows] == [1, 10, 100] @pytest.mark.asyncio From d637ed46762fdbbd8e32b86f258cd9a53c1cfdc7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 11:11:08 -0800 Subject: [PATCH 0554/1866] Use execute_write_many to optimize internal DB, refs #1555, #1570 --- datasette/utils/internal_db.py | 142 +++++++++++++++++++-------------- tests/test_api.py | 2 +- 2 files changed, 81 insertions(+), 63 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 8a145767..95055d8b 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -70,84 +70,102 @@ async def populate_schema_tables(internal_db, db): "DELETE FROM tables WHERE database_name = ?", [database_name], block=True ) tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows + tables_to_insert = [] + columns_to_delete = [] + columns_to_insert = [] + foreign_keys_to_delete = [] + foreign_keys_to_insert = [] + indexes_to_delete = [] + indexes_to_insert = [] + for table in tables: table_name = table["name"] - await internal_db.execute_write( - """ - INSERT INTO tables (database_name, table_name, rootpage, sql) - values (?, ?, ?, ?) - """, - [database_name, table_name, table["rootpage"], table["sql"]], - block=True, - ) - # And the columns - await internal_db.execute_write( - "DELETE FROM columns WHERE database_name = ? and table_name = ?", - [database_name, table_name], - block=True, + tables_to_insert.append( + (database_name, table_name, table["rootpage"], table["sql"]) ) + columns_to_delete.append((database_name, table_name)) columns = await db.table_column_details(table_name) - for column in columns: - params = { + columns_to_insert.extend( + { **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } - await internal_db.execute_write( - """ - INSERT INTO columns ( - database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden - ) VALUES ( - :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden - ) - """, - params, - block=True, - ) - # And the foreign_keys - await internal_db.execute_write( - "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?", - [database_name, table_name], - block=True, + for column in columns ) + foreign_keys_to_delete.append((database_name, table_name)) foreign_keys = ( await db.execute(f"PRAGMA foreign_key_list([{table_name}])") ).rows - for foreign_key in foreign_keys: - params = { + foreign_keys_to_insert.extend( + { **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } - await internal_db.execute_write( - """ - INSERT INTO foreign_keys ( - database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match - ) VALUES ( - :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match - ) - """, - params, - block=True, - ) - # And the indexes - await internal_db.execute_write( - "DELETE FROM indexes WHERE database_name = ? and table_name = ?", - [database_name, table_name], - block=True, + for foreign_key in foreign_keys ) + indexes_to_delete.append((database_name, table_name)) indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows - for index in indexes: - params = { + indexes_to_insert.extend( + { **{"database_name": database_name, "table_name": table_name}, **dict(index), } - await internal_db.execute_write( - """ - INSERT INTO indexes ( - database_name, table_name, seq, name, "unique", origin, partial - ) VALUES ( - :database_name, :table_name, :seq, :name, :unique, :origin, :partial - ) - """, - params, - block=True, - ) + for index in indexes + ) + + await internal_db.execute_write_many( + """ + INSERT INTO tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + tables_to_insert, + block=True, + ) + await internal_db.execute_write_many( + "DELETE FROM columns WHERE database_name = ? and table_name = ?", + columns_to_delete, + block=True, + ) + await internal_db.execute_write_many( + """ + INSERT INTO columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + columns_to_insert, + block=True, + ) + await internal_db.execute_write_many( + "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?", + foreign_keys_to_delete, + block=True, + ) + await internal_db.execute_write_many( + """ + INSERT INTO foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + foreign_keys_to_insert, + block=True, + ) + await internal_db.execute_write_many( + "DELETE FROM indexes WHERE database_name = ? and table_name = ?", + indexes_to_delete, + block=True, + ) + await internal_db.execute_write_many( + """ + INSERT INTO indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + indexes_to_insert, + block=True, + ) diff --git a/tests/test_api.py b/tests/test_api.py index 29c92920..f198c1f9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -937,7 +937,7 @@ def test_trace(trace_debug): assert isinstance(trace["traceback"], list) assert isinstance(trace["database"], str) assert isinstance(trace["sql"], str) - assert isinstance(trace["params"], (list, dict, None.__class__)) + assert isinstance(trace.get("params"), (list, dict, None.__class__)) sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace] # There should be a mix of different types of SQL statement From 97b1723dd09cf000485d4e050efc5bb4f5184a06 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 19:49:11 -0800 Subject: [PATCH 0555/1866] Optimize init_internal_db by running PRAGMA in a single function Refs #1555 --- datasette/utils/internal_db.py | 102 ++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 95055d8b..58f99825 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,4 +1,5 @@ import textwrap +from datasette.utils import table_column_details async def init_internal_db(db): @@ -70,49 +71,70 @@ async def populate_schema_tables(internal_db, db): "DELETE FROM tables WHERE database_name = ?", [database_name], block=True ) tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows - tables_to_insert = [] - columns_to_delete = [] - columns_to_insert = [] - foreign_keys_to_delete = [] - foreign_keys_to_insert = [] - indexes_to_delete = [] - indexes_to_insert = [] - for table in tables: - table_name = table["name"] - tables_to_insert.append( - (database_name, table_name, table["rootpage"], table["sql"]) - ) - columns_to_delete.append((database_name, table_name)) - columns = await db.table_column_details(table_name) - columns_to_insert.extend( - { - **{"database_name": database_name, "table_name": table_name}, - **column._asdict(), - } - for column in columns - ) - foreign_keys_to_delete.append((database_name, table_name)) - foreign_keys = ( - await db.execute(f"PRAGMA foreign_key_list([{table_name}])") - ).rows - foreign_keys_to_insert.extend( - { - **{"database_name": database_name, "table_name": table_name}, - **dict(foreign_key), - } - for foreign_key in foreign_keys - ) - indexes_to_delete.append((database_name, table_name)) - indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows - indexes_to_insert.extend( - { - **{"database_name": database_name, "table_name": table_name}, - **dict(index), - } - for index in indexes + def collect_info(conn): + tables_to_insert = [] + columns_to_delete = [] + columns_to_insert = [] + foreign_keys_to_delete = [] + foreign_keys_to_insert = [] + indexes_to_delete = [] + indexes_to_insert = [] + + for table in tables: + table_name = table["name"] + tables_to_insert.append( + (database_name, table_name, table["rootpage"], table["sql"]) + ) + columns_to_delete.append((database_name, table_name)) + columns = table_column_details(conn, table_name) + columns_to_insert.extend( + { + **{"database_name": database_name, "table_name": table_name}, + **column._asdict(), + } + for column in columns + ) + foreign_keys_to_delete.append((database_name, table_name)) + foreign_keys = conn.execute( + f"PRAGMA foreign_key_list([{table_name}])" + ).fetchall() + foreign_keys_to_insert.extend( + { + **{"database_name": database_name, "table_name": table_name}, + **dict(foreign_key), + } + for foreign_key in foreign_keys + ) + indexes_to_delete.append((database_name, table_name)) + indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() + indexes_to_insert.extend( + { + **{"database_name": database_name, "table_name": table_name}, + **dict(index), + } + for index in indexes + ) + return ( + tables_to_insert, + columns_to_delete, + columns_to_insert, + foreign_keys_to_delete, + foreign_keys_to_insert, + indexes_to_delete, + indexes_to_insert, ) + ( + tables_to_insert, + columns_to_delete, + columns_to_insert, + foreign_keys_to_delete, + foreign_keys_to_insert, + indexes_to_delete, + indexes_to_insert, + ) = await db.execute_fn(collect_info) + await internal_db.execute_write_many( """ INSERT INTO tables (database_name, table_name, rootpage, sql) From c6ff1f23e6a0b26dde8f5b30be3b868b031b6ecf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 18 Dec 2021 20:03:21 -0800 Subject: [PATCH 0556/1866] Queries took rather than query took, closes #1572 --- datasette/templates/_footer.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/_footer.html b/datasette/templates/_footer.html index b1380ae9..074270f1 100644 --- a/datasette/templates/_footer.html +++ b/datasette/templates/_footer.html @@ -1,5 +1,5 @@ Powered by <a href="https://datasette.io/" title="Datasette v{{ datasette_version }}">Datasette</a> -{% if query_ms %}· Query took {{ query_ms|round(3) }}ms{% endif %} +{% if query_ms %}· Queries took {{ query_ms|round(3) }}ms{% endif %} {% if metadata %} {% if metadata.license or metadata.license_url %}· Data license: {% if metadata.license_url %} From f65817000fdf87ce8a0c23edc40784ebe33b5842 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 19 Dec 2021 12:30:34 -0800 Subject: [PATCH 0557/1866] Include count in execute_write_many traces, closes #1571 --- datasette/database.py | 19 +++++++++++++++---- datasette/tracer.py | 6 +++--- tests/test_api.py | 14 +++++++++++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 1de1d5ec..0e41ff32 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -114,11 +114,22 @@ class Database: async def execute_write_many(self, sql, params_seq, block=False): def _inner(conn): - with conn: - return conn.executemany(sql, params_seq) + count = 0 - with trace("sql", database=self.name, sql=sql.strip(), executemany=True): - results = await self.execute_write_fn(_inner, block=block) + def count_params(params): + nonlocal count + for param in params: + count += 1 + yield param + + with conn: + return conn.executemany(sql, count_params(params_seq)), count + + with trace( + "sql", database=self.name, sql=sql.strip(), executemany=True + ) as kwargs: + results, count = await self.execute_write_fn(_inner, block=block) + kwargs["count"] = count return results async def execute_write_fn(self, fn, block=False): diff --git a/datasette/tracer.py b/datasette/tracer.py index 62c3c90c..6703f060 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -32,14 +32,14 @@ def trace(type, **kwargs): ), f".trace() keyword parameters cannot include {TRACE_RESERVED_KEYS}" task_id = get_task_id() if task_id is None: - yield + yield kwargs return tracer = tracers.get(task_id) if tracer is None: - yield + yield kwargs return start = time.perf_counter() - yield + yield kwargs end = time.perf_counter() trace_info = { "type": type, diff --git a/tests/test_api.py b/tests/test_api.py index f198c1f9..8ecaef43 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -928,8 +928,9 @@ def test_trace(trace_debug): assert isinstance(trace_info["sum_trace_duration_ms"], float) assert isinstance(trace_info["num_traces"], int) assert isinstance(trace_info["traces"], list) - assert len(trace_info["traces"]) == trace_info["num_traces"] - for trace in trace_info["traces"]: + traces = trace_info["traces"] + assert len(traces) == trace_info["num_traces"] + for trace in traces: assert isinstance(trace["type"], str) assert isinstance(trace["start"], float) assert isinstance(trace["end"], float) @@ -939,7 +940,7 @@ def test_trace(trace_debug): assert isinstance(trace["sql"], str) assert isinstance(trace.get("params"), (list, dict, None.__class__)) - sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace] + sqls = [trace["sql"] for trace in traces if "sql" in trace] # There should be a mix of different types of SQL statement expected = ( "CREATE TABLE ", @@ -954,6 +955,13 @@ def test_trace(trace_debug): sql.startswith(prefix) for sql in sqls ), "No trace beginning with: {}".format(prefix) + # Should be at least one executescript + assert any(trace for trace in traces if trace.get("executescript")) + # And at least one executemany + execute_manys = [trace for trace in traces if trace.get("executemany")] + assert execute_manys + assert all(isinstance(trace["count"], int) for trace in execute_manys) + @pytest.mark.parametrize( "path,status_code", From 5fac26aa221a111d7633f2dd92014641f7c0ade9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 19 Dec 2021 12:54:12 -0800 Subject: [PATCH 0558/1866] Another populate_schema_tables optimization, refs #1555 --- datasette/utils/internal_db.py | 41 +++++++++------------------------- tests/test_api.py | 1 - 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 58f99825..ed589a7a 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -67,18 +67,23 @@ async def init_internal_db(db): async def populate_schema_tables(internal_db, db): database_name = db.name - await internal_db.execute_write( - "DELETE FROM tables WHERE database_name = ?", [database_name], block=True - ) + + def delete_everything(conn): + conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name]) + conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name]) + conn.execute( + "DELETE FROM foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name]) + + await internal_db.execute_write_fn(delete_everything, block=True) + tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows def collect_info(conn): tables_to_insert = [] - columns_to_delete = [] columns_to_insert = [] - foreign_keys_to_delete = [] foreign_keys_to_insert = [] - indexes_to_delete = [] indexes_to_insert = [] for table in tables: @@ -86,7 +91,6 @@ async def populate_schema_tables(internal_db, db): tables_to_insert.append( (database_name, table_name, table["rootpage"], table["sql"]) ) - columns_to_delete.append((database_name, table_name)) columns = table_column_details(conn, table_name) columns_to_insert.extend( { @@ -95,7 +99,6 @@ async def populate_schema_tables(internal_db, db): } for column in columns ) - foreign_keys_to_delete.append((database_name, table_name)) foreign_keys = conn.execute( f"PRAGMA foreign_key_list([{table_name}])" ).fetchall() @@ -106,7 +109,6 @@ async def populate_schema_tables(internal_db, db): } for foreign_key in foreign_keys ) - indexes_to_delete.append((database_name, table_name)) indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall() indexes_to_insert.extend( { @@ -117,21 +119,15 @@ async def populate_schema_tables(internal_db, db): ) return ( tables_to_insert, - columns_to_delete, columns_to_insert, - foreign_keys_to_delete, foreign_keys_to_insert, - indexes_to_delete, indexes_to_insert, ) ( tables_to_insert, - columns_to_delete, columns_to_insert, - foreign_keys_to_delete, foreign_keys_to_insert, - indexes_to_delete, indexes_to_insert, ) = await db.execute_fn(collect_info) @@ -143,11 +139,6 @@ async def populate_schema_tables(internal_db, db): tables_to_insert, block=True, ) - await internal_db.execute_write_many( - "DELETE FROM columns WHERE database_name = ? and table_name = ?", - columns_to_delete, - block=True, - ) await internal_db.execute_write_many( """ INSERT INTO columns ( @@ -159,11 +150,6 @@ async def populate_schema_tables(internal_db, db): columns_to_insert, block=True, ) - await internal_db.execute_write_many( - "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?", - foreign_keys_to_delete, - block=True, - ) await internal_db.execute_write_many( """ INSERT INTO foreign_keys ( @@ -175,11 +161,6 @@ async def populate_schema_tables(internal_db, db): foreign_keys_to_insert, block=True, ) - await internal_db.execute_write_many( - "DELETE FROM indexes WHERE database_name = ? and table_name = ?", - indexes_to_delete, - block=True, - ) await internal_db.execute_write_many( """ INSERT INTO indexes ( diff --git a/tests/test_api.py b/tests/test_api.py index 8ecaef43..574ebb41 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -946,7 +946,6 @@ def test_trace(trace_debug): "CREATE TABLE ", "PRAGMA ", "INSERT OR REPLACE INTO ", - "DELETE FROM ", "INSERT INTO", "select ", ) From 4094741c2881c2ada3f3f878b532fdaec7914953 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 19 Dec 2021 13:11:57 -0800 Subject: [PATCH 0559/1866] Fixed bug with custom templates for writable canned queries, closes #1547 --- datasette/views/database.py | 11 +++++------ tests/test_canned_queries.py | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index f1901b34..aa8d27ec 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -246,6 +246,11 @@ class QueryView(DataView): extra_args["page_size"] = _size templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) query_error = None @@ -340,12 +345,6 @@ class QueryView(DataView): results = None columns = [] - if canned_query: - templates.insert( - 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", - ) - allow_execute_sql = await self.ds.permission_allowed( request.actor, "execute-sql", database, default=True ) diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index cea81ec7..c5ccaf5c 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -6,9 +6,19 @@ from .fixtures import make_app_client, app_client @pytest.fixture -def canned_write_client(): +def canned_write_client(tmpdir): + template_dir = tmpdir / "canned_write_templates" + template_dir.mkdir() + (template_dir / "query-data-update_name.html").write_text( + """ + {% extends "query.html" %} + {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} + """, + "utf-8", + ) with make_app_client( extra_databases={"data.db": "create table names (name text)"}, + template_dir=str(template_dir), metadata={ "databases": { "data": { @@ -344,3 +354,13 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c ) assert 400 == response.status assert response.json["error"].startswith("You did not supply a value for binding") + + +def test_canned_write_custom_template(canned_write_client): + response = canned_write_client.get("/data/update_name") + assert response.status == 200 + assert ( + "<!-- Templates considered: *query-data-update_name.html, query-data.html, query.html -->" + in response.text + ) + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text From dbaac79946034e0b00714e2da39f934d693883d2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 19 Dec 2021 14:08:10 -0800 Subject: [PATCH 0560/1866] Release 0.60a1 Refs #1547, #1555, #1562, #1563, #1564, #1567, #1568, #1569, #1570, #1571, #1572 --- datasette/version.py | 2 +- docs/changelog.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 2fce006c..290fbcf3 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.60a0" +__version__ = "0.60a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 92a9d941..99d3315e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog ========= +.. _v0_60a1: + +0.60a1 (2021-12-19) +------------------- + +- Database write connections now execute the :ref:`plugin_hook_prepare_connection` plugin hook. (:issue:`1564`) +- The ``Datasette()`` constructor no longer requires the ``files=`` argument, and is now documented at :ref:`internals_datasette`. (:issue:`1563`) +- The tracing feature now traces write queries, not just read queries. (:issue:`1568`) +- Added two methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`) +- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`) +- Fixed bug where writable canned queries could not be used with custom templates. (:issue:`1547`) + .. _v0_60a0: 0.60a0 (2021-12-17) From f36e010b3b69ada104b79d83c7685caf9359049e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 19 Dec 2021 17:25:40 -0800 Subject: [PATCH 0561/1866] Upgrade to Pluggy>=1.0, refs #1575 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8cd3e5b..1f8855cf 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "hupper~=1.9", "httpx>=0.20", "pint~=0.9", - "pluggy>=0.13,<1.1", + "pluggy>=1.0,<1.1", "uvicorn~=0.11", "aiofiles>=0.4,<0.9", "janus>=0.6.2,<1.1", From 554aae5c51b7ce1b570e0e9bbe45f4cf9200f2bf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 20 Dec 2021 09:23:05 -0800 Subject: [PATCH 0562/1866] Plausible analytics for the documentation --- docs/_templates/layout.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index b7b6f794..e44f3b56 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,5 +1,10 @@ {%- extends "!layout.html" %} +{% block htmltitle %} +{{ super() }} +<script defer data-domain="datasette.io" src="https://plausible.io/js/plausible.js"></script> +{% endblock %} + {% block sidebartitle %} <a href="https://datasette.io/"> From 6b1384b2f529134998fb507e63307609a5b7f5c0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 20 Dec 2021 15:55:17 -0800 Subject: [PATCH 0563/1866] Track plausible for docs.datasette.io not datasette.io --- docs/_templates/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index e44f3b56..db16b428 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -2,7 +2,7 @@ {% block htmltitle %} {{ super() }} -<script defer data-domain="datasette.io" src="https://plausible.io/js/plausible.js"></script> +<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script> {% endblock %} {% block sidebartitle %} From ace86566b28280091b3844cf5fbecd20158e9004 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 22 Dec 2021 12:22:44 -0800 Subject: [PATCH 0564/1866] Remove concept of special_args, re-arrange TableView a bit, refs #1518 --- datasette/views/table.py | 79 +++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index c3bcf01d..9808fd24 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -366,6 +366,30 @@ class TableView(RowTableShared): None, "view-table", (database, table), default=True ) + # Handle ?_filter_column and redirect, if present + redirect_params = filters_should_redirect(request.args) + if redirect_params: + return self.redirect( + request, + path_with_added_args(request, redirect_params), + forward_querystring=False, + ) + + # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort) + if "_sort_by_desc" in request.args: + return self.redirect( + request, + path_with_added_args( + request, + { + "_sort_desc": request.args.get("_sort"), + "_sort_by_desc": None, + "_sort": None, + }, + ), + forward_querystring=False, + ) + # Introspect columns and primary keys for table pks = await db.primary_keys(table) table_columns = await db.table_columns(table) @@ -399,47 +423,20 @@ class TableView(RowTableShared): nocount = True nofacet = True - # Special args start with _ and do not contain a __ - # That's so if there is a column that starts with _ - # it can still be queried using ?_col__exact=blah - special_args = {} - other_args = [] - for key in request.args: - if key.startswith("_") and "__" not in key: - special_args[key] = request.args[key] - else: - for v in request.args.getlist(key): - other_args.append((key, v)) - - # Handle ?_filter_column and redirect, if present - redirect_params = filters_should_redirect(special_args) - if redirect_params: - return self.redirect( - request, - path_with_added_args(request, redirect_params), - forward_querystring=False, - ) - - # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort) - if "_sort_by_desc" in special_args: - return self.redirect( - request, - path_with_added_args( - request, - { - "_sort_desc": special_args.get("_sort"), - "_sort_by_desc": None, - "_sort": None, - }, - ), - forward_querystring=False, - ) - table_metadata = self.ds.table_metadata(database, table) units = table_metadata.get("units", {}) + # Arguments that start with _ and don't contain a __ are + # special - things like ?_search= - and should not be + # treated as filters. + filter_args = [] + for key in request.args: + if not (key.startswith("_") and "__" not in key): + for v in request.args.getlist(key): + filter_args.append((key, v)) + # Build where clauses from query string arguments - filters = Filters(sorted(other_args), units, ureg) + filters = Filters(sorted(filter_args), units, ureg) where_clauses, params = filters.build_where_clauses(table) # Execute filters_from_request plugin hooks - including the default @@ -464,8 +461,8 @@ class TableView(RowTableShared): sortable_columns = await self.sortable_columns_for_table( database, table, use_rowid ) - sort = special_args.get("_sort") - sort_desc = special_args.get("_sort_desc") + sort = request.args.get("_sort") + sort_desc = request.args.get("_sort_desc") if not sort and not sort_desc: sort = table_metadata.get("sort") @@ -498,7 +495,7 @@ class TableView(RowTableShared): count_sql = f"select count(*) {from_sql}" # Handle pagination driven by ?_next= - _next = _next or special_args.get("_next") + _next = _next or request.args.get("_next") offset = "" if _next: sort_value = None @@ -708,7 +705,7 @@ class TableView(RowTableShared): expandable_columns = await self.expandable_columns(database, table) columns_to_expand = None try: - all_labels = value_as_boolean(special_args.get("_labels", "")) + all_labels = value_as_boolean(request.args.get("_labels", "")) except ValueError: all_labels = default_labels # Check for explicit _label= From 00a2895cd2dc42c63846216b36b2dc9f41170129 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 23 Dec 2021 11:03:49 -0800 Subject: [PATCH 0565/1866] execute_write defaut is now block=True, closes #1579 --- datasette/database.py | 8 ++++---- docs/internals.rst | 14 +++++++------- tests/test_internals_database.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 0e41ff32..e908d1ea 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -94,7 +94,7 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False ) - async def execute_write(self, sql, params=None, block=False): + async def execute_write(self, sql, params=None, block=True): def _inner(conn): with conn: return conn.execute(sql, params or []) @@ -103,7 +103,7 @@ class Database: results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_script(self, sql, block=False): + async def execute_write_script(self, sql, block=True): def _inner(conn): with conn: return conn.executescript(sql) @@ -112,7 +112,7 @@ class Database: results = await self.execute_write_fn(_inner, block=block) return results - async def execute_write_many(self, sql, params_seq, block=False): + async def execute_write_many(self, sql, params_seq, block=True): def _inner(conn): count = 0 @@ -132,7 +132,7 @@ class Database: kwargs["count"] = count return results - async def execute_write_fn(self, fn, block=False): + async def execute_write_fn(self, fn, block=True): task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") if self._write_queue is None: self._write_queue = queue.Queue() diff --git a/docs/internals.rst b/docs/internals.rst index bc0174a8..667ac33a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -663,7 +663,7 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, block=False) +await db.execute_write(sql, params=None, block=True) ----------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -672,20 +672,20 @@ This method can be used to queue up a non-SELECT SQL query to be executed agains You can pass additional SQL parameters as a tuple or dictionary. -By default queries are considered to be "fire and forget" - they will be added to the queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. +The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. -If you pass ``block=True`` this behaviour changes: the method will block until the write operation has completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. +If you pass ``block=False`` this behaviour changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. .. _database_execute_write_script: -await db.execute_write_script(sql, block=False) +await db.execute_write_script(sql, block=True) ----------------------------------------------- Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executescript>`__ method. .. _database_execute_write_many: -await db.execute_write_many(sql, params_seq, block=False) +await db.execute_write_many(sql, params_seq, block=True) --------------------------------------------------------- Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Cursor.executemany>`__ method. This will efficiently execute the same SQL statement against each of the parameters in the ``params_seq`` iterator, for example: @@ -700,7 +700,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d .. _database_execute_write_fn: -await db.execute_write_fn(fn, block=False) +await db.execute_write_fn(fn, block=True) ------------------------------------------ This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. This function will be queued up and then called when the write connection is available, passing that connection as the argument to the function. @@ -725,7 +725,7 @@ This method is fire-and-forget, queueing your function to be executed and then a If you pass ``block=True`` your calling code will block until the function has been executed. The return value to the ``await`` will be the return value of your function. -If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=False`` any exceptions will be silently ignored. +If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=True`` any exceptions will be silently ignored. Here's an example of ``block=True`` in action: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index f751bf9a..80f47ab9 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -435,7 +435,7 @@ async def test_execute_write_fn_block_false(db): row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] - task_id = await db.execute_write_fn(write_fn) + task_id = await db.execute_write_fn(write_fn, block=False) assert isinstance(task_id, uuid.UUID) From 75153ea9b94d09ec3d61f7c6ebdf378e0c0c7a0b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 23 Dec 2021 11:16:31 -0800 Subject: [PATCH 0566/1866] Updated db.execute_write_fn() docs for block=True default, refs #1579 --- docs/internals.rst | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 667ac33a..6a5666fd 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -331,7 +331,7 @@ This will add a mutable database and serve it at ``/my-new-database``. .. code-block:: python db = datasette.add_database(Database(datasette, memory_name="statistics")) - await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) + await db.execute_write("CREATE TABLE foo(id integer primary key)") .. _datasette_add_memory_database: @@ -694,8 +694,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d await db.execute_write_many( "insert into characters (id, name) values (?, ?)", - [(1, "Melanie"), (2, "Selma"), (2, "Viktor")], - block=True, + [(1, "Melanie"), (2, "Selma"), (2, "Viktor")] ) .. _database_execute_write_fn: @@ -703,9 +702,9 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d await db.execute_write_fn(fn, block=True) ------------------------------------------ -This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. This function will be queued up and then called when the write connection is available, passing that connection as the argument to the function. +This method works like ``.execute_write()``, but instead of a SQL statement you give it a callable Python function. Your function will be queued up and then called when the write connection is available, passing that connection as the argument to the function. -The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection as long as it is executing. +The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection for as long as it is executing. .. warning:: @@ -715,31 +714,21 @@ For example: .. code-block:: python - def my_action(conn): - conn.execute("delete from some_table") - conn.execute("delete from other_table") - - await database.execute_write_fn(my_action) - -This method is fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. - -If you pass ``block=True`` your calling code will block until the function has been executed. The return value to the ``await`` will be the return value of your function. - -If your function raises an exception and you specified ``block=True``, that exception will be propagated up to the ``await`` line. With ``block=True`` any exceptions will be silently ignored. - -Here's an example of ``block=True`` in action: - -.. code-block:: python - - def my_action(conn): + def delete_and_return_count(conn): conn.execute("delete from some_table where id > 5") return conn.execute("select count(*) from some_table").fetchone()[0] try: - num_rows_left = await database.execute_write_fn(my_action, block=True) + num_rows_left = await database.execute_write_fn(delete_and_return_count) except Exception as e: print("An error occurred:", e) +The value returned from ``await database.execute_write_fn(...)`` will be the return value from your function. + +If your function raises an exception that exception will be propagated up to the ``await`` line. + +If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. + .. _internals_database_introspection: Database introspection From 8c401ee0f054de2f568c3a8302c9223555146407 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 23 Dec 2021 11:18:20 -0800 Subject: [PATCH 0567/1866] Fixed remaining code and docs for new block=True default, closes #1579 --- datasette/app.py | 1 - datasette/utils/internal_db.py | 8 ++------ datasette/views/database.py | 2 +- docs/plugin_hooks.rst | 2 +- tests/test_facets.py | 17 +++++------------ tests/test_internals_database.py | 28 ++++++++++------------------ 6 files changed, 19 insertions(+), 39 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 17fa06a5..bd663509 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -367,7 +367,6 @@ class Datasette: VALUES (?, ?, ?, ?) """, [database_name, str(db.path), db.is_memory, schema_version], - block=True, ) await populate_schema_tables(internal_db, db) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index ed589a7a..e4b49e80 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -62,7 +62,7 @@ async def init_internal_db(db): ); """ ).strip() - await db.execute_write_script(create_tables_sql, block=True) + await db.execute_write_script(create_tables_sql) async def populate_schema_tables(internal_db, db): @@ -76,7 +76,7 @@ async def populate_schema_tables(internal_db, db): ) conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name]) - await internal_db.execute_write_fn(delete_everything, block=True) + await internal_db.execute_write_fn(delete_everything) tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows @@ -137,7 +137,6 @@ async def populate_schema_tables(internal_db, db): values (?, ?, ?, ?) """, tables_to_insert, - block=True, ) await internal_db.execute_write_many( """ @@ -148,7 +147,6 @@ async def populate_schema_tables(internal_db, db): ) """, columns_to_insert, - block=True, ) await internal_db.execute_write_many( """ @@ -159,7 +157,6 @@ async def populate_schema_tables(internal_db, db): ) """, foreign_keys_to_insert, - block=True, ) await internal_db.execute_write_many( """ @@ -170,5 +167,4 @@ async def populate_schema_tables(internal_db, db): ) """, indexes_to_insert, - block=True, ) diff --git a/datasette/views/database.py b/datasette/views/database.py index aa8d27ec..e26706e7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -279,7 +279,7 @@ class QueryView(DataView): ok = None try: cursor = await self.ds.databases[database].execute_write( - sql, params_for_query, block=True + sql, params_for_query ) message = metadata.get( "on_success_message" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index cbaf4c54..88e1def0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -770,7 +770,7 @@ Or you can return an async function which will be awaited on startup. Use this o if "my_table" not in await db.table_names(): await db.execute_write(""" create table my_table (mycol text) - """, block=True) + """) return inner Potential use-cases: diff --git a/tests/test_facets.py b/tests/test_facets.py index 3f292a3b..c28dc43c 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -408,16 +408,14 @@ async def test_array_facet_results(app_client): async def test_array_facet_handle_duplicate_tags(): ds = Datasette([], memory=True) db = ds.add_database(Database(ds, memory_name="test_array_facet")) - await db.execute_write("create table otters(name text, tags text)", block=True) + await db.execute_write("create table otters(name text, tags text)") for name, tags in ( ("Charles", ["friendly", "cunning", "friendly"]), ("Shaun", ["cunning", "empathetic", "friendly"]), ("Tracy", ["empathetic", "eager"]), ): await db.execute_write( - "insert into otters (name, tags) values (?, ?)", - [name, json.dumps(tags)], - block=True, + "insert into otters (name, tags) values (?, ?)", [name, json.dumps(tags)] ) response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags") @@ -516,11 +514,9 @@ async def test_date_facet_results(app_client): async def test_json_array_with_blanks_and_nulls(): ds = Datasette([], memory=True) db = ds.add_database(Database(ds, memory_name="test_json_array")) - await db.execute_write("create table foo(json_column text)", block=True) + await db.execute_write("create table foo(json_column text)") for value in ('["a", "b", "c"]', '["a", "b"]', "", None): - await db.execute_write( - "insert into foo (json_column) values (?)", [value], block=True - ) + await db.execute_write("insert into foo (json_column) values (?)", [value]) response = await ds.client.get("/test_json_array/foo.json") data = response.json() assert data["suggested_facets"] == [ @@ -536,15 +532,12 @@ async def test_json_array_with_blanks_and_nulls(): async def test_facet_size(): ds = Datasette([], memory=True, settings={"max_returned_rows": 50}) db = ds.add_database(Database(ds, memory_name="test_facet_size")) - await db.execute_write( - "create table neighbourhoods(city text, neighbourhood text)", block=True - ) + await db.execute_write("create table neighbourhoods(city text, neighbourhood text)") for i in range(1, 51): for j in range(1, 4): await db.execute_write( "insert into neighbourhoods (city, neighbourhood) values (?, ?)", ["City {}".format(i), "Neighbourhood {}".format(j)], - block=True, ) response = await ds.client.get("/test_facet_size/neighbourhoods.json") data = response.json() diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 80f47ab9..bcecb486 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -377,9 +377,7 @@ async def test_table_names(db): @pytest.mark.asyncio async def test_execute_write_block_true(db): await db.execute_write( - "update roadside_attractions set name = ? where pk = ?", - ["Mystery!", 1], - block=True, + "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1] ) rows = await db.execute("select name from roadside_attractions where pk = 1") assert "Mystery!" == rows.rows[0][0] @@ -399,8 +397,7 @@ async def test_execute_write_block_false(db): @pytest.mark.asyncio async def test_execute_write_script(db): await db.execute_write_script( - "create table foo (id integer primary key); create table bar (id integer primary key); ", - block=True, + "create table foo (id integer primary key); create table bar (id integer primary key);" ) table_names = await db.table_names() assert {"foo", "bar"}.issubset(table_names) @@ -408,14 +405,9 @@ async def test_execute_write_script(db): @pytest.mark.asyncio async def test_execute_write_many(db): - await db.execute_write_script( - "create table foomany (id integer primary key)", - block=True, - ) + await db.execute_write_script("create table foomany (id integer primary key)") await db.execute_write_many( - "insert into foomany (id) values (?)", - [(1,), (10,), (100,)], - block=True, + "insert into foomany (id) values (?)", [(1,), (10,), (100,)] ) result = await db.execute("select * from foomany") assert [r[0] for r in result.rows] == [1, 10, 100] @@ -424,7 +416,7 @@ async def test_execute_write_many(db): @pytest.mark.asyncio async def test_execute_write_has_correctly_prepared_connection(db): # The sleep() function is only available if ds._prepare_connection() was called - await db.execute_write("select sleep(0.01)", block=True) + await db.execute_write("select sleep(0.01)") @pytest.mark.asyncio @@ -447,7 +439,7 @@ async def test_execute_write_fn_block_true(db): row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] - new_count = await db.execute_write_fn(write_fn, block=True) + new_count = await db.execute_write_fn(write_fn) assert 3 == new_count @@ -457,7 +449,7 @@ async def test_execute_write_fn_exception(db): assert False with pytest.raises(AssertionError): - await db.execute_write_fn(write_fn, block=True) + await db.execute_write_fn(write_fn) @pytest.mark.asyncio @@ -472,7 +464,7 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): assert False with pytest.raises(AssertionError): - await db.execute_write_fn(write_fn, block=True) + await db.execute_write_fn(write_fn) app_client.ds.remove_database("immutable-db") @@ -513,7 +505,7 @@ async def test_database_memory_name(app_client): table_names = await db.table_names() assert table_names == [] # Now create a table in foo - await foo1.execute_write("create table foo (t text)", block=True) + await foo1.execute_write("create table foo (t text)") assert await foo1.table_names() == ["foo"] assert await foo2.table_names() == ["foo"] assert await bar1.table_names() == [] @@ -528,5 +520,5 @@ async def test_in_memory_databases_forbid_writes(app_client): await db.execute("create table foo (t text)") assert await db.table_names() == [] # Using db.execute_write() should work: - await db.execute_write("create table foo (t text)", block=True) + await db.execute_write("create table foo (t text)") assert await db.table_names() == ["foo"] From 63537dd3decfd59636f4a42b336785ef49f0cec0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 12:34:55 -0800 Subject: [PATCH 0568/1866] Allow 'explain query plan' with more whitespace, closes #1588 --- datasette/utils/__init__.py | 8 ++++---- tests/test_utils.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c339113c..bc3155a5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -162,11 +162,11 @@ class InvalidSql(Exception): allowed_sql_res = [ re.compile(r"^select\b"), - re.compile(r"^explain select\b"), - re.compile(r"^explain query plan select\b"), + re.compile(r"^explain\s+select\b"), + re.compile(r"^explain\s+query\s+plan\s+select\b"), re.compile(r"^with\b"), - re.compile(r"^explain with\b"), - re.compile(r"^explain query plan with\b"), + re.compile(r"^explain\s+with\b"), + re.compile(r"^explain\s+query\s+plan\s+with\b"), ] allowed_pragmas = ( "database_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index e1b61072..e7d67045 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -157,7 +157,9 @@ def test_validate_sql_select_bad(bad_sql): "select '# Hello there\n\n* This is a list\n* of items\n--\n[And a link](https://github.com/simonw/datasette-render-markdown).'\nas demo_markdown", "select 1 + 1", "explain select 1 + 1", + "explain\nselect 1 + 1", "explain query plan select 1 + 1", + "explain query plan\nselect 1 + 1", "SELECT\nblah FROM foo", "WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", "explain WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;", From 4b23f01f3e668c8f2a2f1a294be49f49b4073969 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 13:35:54 -0800 Subject: [PATCH 0569/1866] CLI reference docs, maintained by cog - refs #1594 --- .github/workflows/test.yml | 3 + docs/cli-reference.rst | 355 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + setup.py | 1 + 4 files changed, 360 insertions(+) create mode 100644 docs/cli-reference.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b3635fe..704931a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,3 +28,6 @@ jobs: run: | pytest -n auto -m "not serial" pytest -m "serial" + - name: Check if cog needs to be run + run: | + cog --check docs/*.rst diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst new file mode 100644 index 00000000..adf89633 --- /dev/null +++ b/docs/cli-reference.rst @@ -0,0 +1,355 @@ +.. _cli_reference: + +=============== + CLI reference +=============== + +This page lists the ``--help`` for every ``datasette`` CLI command. + +.. [[[cog + from datasette import cli + from click.testing import CliRunner + import textwrap + commands = [ + ["--help"], + ["serve", "--help"], + ["serve", "--help-settings"], + ["plugins", "--help"], + ["publish", "--help"], + ["publish", "cloudrun", "--help"], + ["publish", "heroku", "--help"], + ["package", "--help"], + ["inspect", "--help"], + ["install", "--help"], + ["uninstall", "--help"], + ] + for command in commands: + title = "datasette " + " ".join(command) + cog.out(title + "\n") + cog.out(("=" * len(title)) + "\n\n") + cog.out("::\n\n") + result = CliRunner().invoke(cli.cli, command) + output = result.output.replace("Usage: cli ", "Usage: datasette ") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] +datasette --help +================ + +:: + + Usage: datasette [OPTIONS] COMMAND [ARGS]... + + Datasette is an open source multi-tool for exploring and publishing data + + About Datasette: https://datasette.io/ + Full documentation: https://docs.datasette.io/ + + Options: + --version Show the version and exit. + --help Show this message and exit. + + Commands: + serve* Serve up specified SQLite database files with a web UI + inspect + install Install Python packages - e.g. + package Package specified SQLite files into a new datasette Docker... + plugins List currently available plugins + publish Publish specified SQLite database files to the internet along... + uninstall Uninstall Python packages (e.g. + + +datasette serve --help +====================== + +:: + + Usage: datasette serve [OPTIONS] [FILES]... + + Serve up specified SQLite database files with a web UI + + Options: + -i, --immutable PATH Database files to open in immutable mode + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means + only connections from the local machine will be + allowed. Use 0.0.0.0 to listen to all IPs and allow + access from other machines. + -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to + automatically assign an available port. + [0<=x<=65535] + --uds TEXT Bind to a Unix domain socket + --reload Automatically reload if code or metadata change + detected - useful for development + --cors Enable CORS by serving Access-Control-Allow-Origin: + * + --load-extension TEXT Path to a SQLite extension to load + --inspect-file TEXT Path to JSON file created using "datasette inspect" + -m, --metadata FILENAME Path to JSON/YAML file containing license/source + metadata + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... + --memory Make /_memory database available + --config CONFIG Deprecated: set config option using + configname:value. Use --setting instead. + --setting SETTING... Setting, see docs.datasette.io/en/stable/config.html + --secret TEXT Secret used for signing secure values, such as + signed cookies + --root Output URL that sets a cookie authenticating the + root user + --get TEXT Run an HTTP GET request against this path, print + results and exit + --version-note TEXT Additional note to show on /-/versions + --help-settings Show available settings + --pdb Launch debugger on any errors + -o, --open Open Datasette in your web browser + --create Create database files if they do not exist + --crossdb Enable cross-database joins using the /_memory + database + --ssl-keyfile TEXT SSL key file + --ssl-certfile TEXT SSL certificate file + --help Show this message and exit. + + +datasette serve --help-settings +=============================== + +:: + + Settings: + default_page_size Default page size for the table view + (default=100) + max_returned_rows Maximum rows that can be returned from a table or + custom query (default=1000) + num_sql_threads Number of threads in the thread pool for + executing SQLite queries (default=3) + sql_time_limit_ms Time limit for a SQL query in milliseconds + (default=1000) + default_facet_size Number of values to return for requested facets + (default=30) + facet_time_limit_ms Time limit for calculating a requested facet + (default=200) + facet_suggest_time_limit_ms Time limit for calculating a suggested facet + (default=50) + hash_urls Include DB file contents hash in URLs, for far- + future caching (default=False) + allow_facet Allow users to specify columns to facet using + ?_facet= parameter (default=True) + allow_download Allow users to download the original SQLite + database files (default=True) + suggest_facets Calculate and display suggested facets + (default=True) + default_cache_ttl Default HTTP cache TTL (used in Cache-Control: + max-age= header) (default=5) + default_cache_ttl_hashed Default HTTP cache TTL for hashed URL pages + (default=31536000) + cache_size_kb SQLite cache size in KB (0 == use SQLite default) + (default=0) + allow_csv_stream Allow .csv?_stream=1 to download all rows + (ignoring max_returned_rows) (default=True) + max_csv_mb Maximum size allowed for CSV export in MB - set 0 + to disable this limit (default=100) + truncate_cells_html Truncate cells longer than this in HTML table + view - set 0 to disable (default=2048) + force_https_urls Force URLs in API output to always use https:// + protocol (default=False) + template_debug Allow display of template debug information with + ?_context=1 (default=False) + trace_debug Allow display of SQL trace debug information with + ?_trace=1 (default=False) + base_url Datasette URLs should use this base path + (default=/) + + + +datasette plugins --help +======================== + +:: + + Usage: datasette plugins [OPTIONS] + + List currently available plugins + + Options: + --all Include built-in default plugins + --plugins-dir DIRECTORY Path to directory containing custom plugins + --help Show this message and exit. + + +datasette publish --help +======================== + +:: + + Usage: datasette publish [OPTIONS] COMMAND [ARGS]... + + Publish specified SQLite database files to the internet along with a + Datasette-powered interface and API + + Options: + --help Show this message and exit. + + Commands: + cloudrun + heroku + + +datasette publish cloudrun --help +================================= + +:: + + Usage: datasette publish cloudrun [OPTIONS] [FILES]... + + Options: + -m, --metadata FILENAME Path to JSON/YAML file containing metadata to + publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. + main + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at + /MOUNT/... + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret <TEXT TEXT TEXT>... + Secrets to pass to plugins, e.g. --plugin- + secret datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --secret TEXT Secret used for signing secure values, such as + signed cookies + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when building + --service TEXT Cloud Run service to deploy (or over-write) + --spatialite Enable SpatialLite extension + --show-files Output the generated Dockerfile and + metadata.json + --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi + --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run + --apt-get-install TEXT Additional packages to apt-get install + --help Show this message and exit. + + +datasette publish heroku --help +=============================== + +:: + + Usage: datasette publish heroku [OPTIONS] [FILES]... + + Options: + -m, --metadata FILENAME Path to JSON/YAML file containing metadata to + publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. + main + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at + /MOUNT/... + --install TEXT Additional packages (e.g. plugins) to install + --plugin-secret <TEXT TEXT TEXT>... + Secrets to pass to plugins, e.g. --plugin- + secret datasette-auth-github client_id xxx + --version-note TEXT Additional note to show on /-/versions + --secret TEXT Secret used for signing secure values, such as + signed cookies + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + -n, --name TEXT Application name to use when deploying + --tar TEXT --tar option to pass to Heroku, e.g. + --tar=/usr/local/bin/gtar + --help Show this message and exit. + + +datasette package --help +======================== + +:: + + Usage: datasette package [OPTIONS] FILES... + + Package specified SQLite files into a new datasette Docker container + + Options: + -t, --tag TEXT Name for the resulting Docker container, can + optionally use name:tag format + -m, --metadata FILENAME Path to JSON/YAML file containing metadata to + publish + --extra-options TEXT Extra options to pass to datasette serve + --branch TEXT Install datasette from a GitHub branch e.g. main + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... + --install TEXT Additional packages (e.g. plugins) to install + --spatialite Enable SpatialLite extension + --version-note TEXT Additional note to show on /-/versions + --secret TEXT Secret used for signing secure values, such as + signed cookies + -p, --port INTEGER RANGE Port to run the server on, defaults to 8001 + [1<=x<=65535] + --title TEXT Title for metadata + --license TEXT License label for metadata + --license_url TEXT License URL for metadata + --source TEXT Source label for metadata + --source_url TEXT Source URL for metadata + --about TEXT About label for metadata + --about_url TEXT About URL for metadata + --help Show this message and exit. + + +datasette inspect --help +======================== + +:: + + Usage: datasette inspect [OPTIONS] [FILES]... + + Options: + --inspect-file TEXT + --load-extension TEXT Path to a SQLite extension to load + --help Show this message and exit. + + +datasette install --help +======================== + +:: + + Usage: datasette install [OPTIONS] PACKAGES... + + Install Python packages - e.g. Datasette plugins - into the same environment + as Datasette + + Options: + -U, --upgrade Upgrade packages to latest version + --help Show this message and exit. + + +datasette uninstall --help +========================== + +:: + + Usage: datasette uninstall [OPTIONS] PACKAGES... + + Uninstall Python packages (e.g. plugins) from the Datasette environment + + Options: + -y, --yes Don't ask for confirmation + --help Show this message and exit. + + +.. [[[end]]] diff --git a/docs/index.rst b/docs/index.rst index eafc5bdb..36e42848 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,4 +60,5 @@ Contents testing_plugins internals contributing + cli-reference changelog diff --git a/setup.py b/setup.py index 1f8855cf..e9ef082a 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ setup( "black==21.12b0", "pytest-timeout>=1.4.2,<2.1", "trustme>=0.7,<0.10", + "cogapp>=3.3.0", ], "rich": ["rich"], }, From 5698e2af0182677c0f1f7f5b3bc61415bb6c93f8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 13:55:13 -0800 Subject: [PATCH 0570/1866] Promote Datasette Desktop in installation docs, closes #1466 --- docs/installation.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 723f1e3f..ac3dcca2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -7,8 +7,7 @@ .. note:: If you just want to try Datasette out you don't need to install anything: see :ref:`getting_started_glitch` -There are two main options for installing Datasette. You can install it directly -on to your machine, or you can install it using Docker. +There are two main options for installing Datasette. You can install it directly on to your machine, or you can install it using Docker. If you want to start making contributions to the Datasette project by installing a copy that lets you directly modify the code, take a look at our guide to :ref:`devenvironment`. @@ -20,6 +19,13 @@ If you want to start making contributions to the Datasette project by installing Basic installation ================== +.. _installation_datasette_desktop: + +Datasette Desktop for Mac +------------------------- + +`Datasette Desktop <https://datasette.io/desktop>`__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line. + .. _installation_homebrew: Using Homebrew From 3658e57ac2de0bec0ea5de36e3ddd09784ecf65e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 14:20:07 -0800 Subject: [PATCH 0571/1866] Fixed bug with table title element, closes #1560 --- datasette/templates/table.html | 3 +-- tests/test_table_html.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index f3749b57..e3c6f38d 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,7 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %} - {% if human_description_en %}where {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{ super() }} diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 50d679a0..f68e05a5 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1038,3 +1038,22 @@ def test_sort_rowid_with_next(app_client): def assert_querystring_equal(expected, actual): assert sorted(expected.split("&")) == sorted(actual.split("&")) + + +@pytest.mark.parametrize( + "path,expected", + ( + ( + "/fixtures/facetable", + "fixtures: facetable: 15 rows", + ), + ( + "/fixtures/facetable?on_earth__exact=1", + "fixtures: facetable: 14 rows where on_earth = 1", + ), + ), +) +def test_table_page_title(app_client, path, expected): + response = app_client.get(path) + title = Soup(response.text, "html.parser").find("title").text + assert title == expected From 88bc2ceae1151ec859f477d527b40f7e36012017 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:07:30 -0800 Subject: [PATCH 0572/1866] --help summary for 'datasette inspect', closes #1597 --- datasette/cli.py | 6 ++++++ docs/cli-reference.rst | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/cli.py b/datasette/cli.py index 22e2338a..12d3d728 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -136,6 +136,12 @@ def cli(): @click.option("--inspect-file", default="-") @sqlite_extensions def inspect(files, inspect_file, sqlite_extensions): + """ + Generate JSON summary of provided database files + + This can then be passed to "datasette --inspect-file" to speed up count + operations against immutable database files. + """ app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) loop = asyncio.get_event_loop() inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions)) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index adf89633..7ac6debe 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -51,7 +51,7 @@ datasette --help Commands: serve* Serve up specified SQLite database files with a web UI - inspect + inspect Generate JSON summary of provided database files install Install Python packages - e.g. package Package specified SQLite files into a new datasette Docker... plugins List currently available plugins @@ -317,6 +317,11 @@ datasette inspect --help Usage: datasette inspect [OPTIONS] [FILES]... + Generate JSON summary of provided database files + + This can then be passed to "datasette --inspect-file" to speed up count + operations against immutable database files. + Options: --inspect-file TEXT --load-extension TEXT Path to a SQLite extension to load From 8f5c44a1669427019b288f5b5debec67a90f908b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:09:38 -0800 Subject: [PATCH 0573/1866] Better --help summaries for install and uninstall --- datasette/cli.py | 4 ++-- docs/cli-reference.rst | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 12d3d728..18054448 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -307,7 +307,7 @@ def package( "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" ) def install(packages, upgrade): - """Install Python packages - e.g. Datasette plugins - into the same environment as Datasette""" + """Install plugins and packages from PyPI into the same environment as Datasette""" args = ["pip", "install"] if upgrade: args += ["--upgrade"] @@ -320,7 +320,7 @@ def install(packages, upgrade): @click.argument("packages", nargs=-1, required=True) @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") def uninstall(packages, yes): - """Uninstall Python packages (e.g. plugins) from the Datasette environment""" + """Uninstall plugins and Python packages from the Datasette environment""" sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) run_module("pip", run_name="__main__") diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7ac6debe..37a30606 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -52,11 +52,11 @@ datasette --help Commands: serve* Serve up specified SQLite database files with a web UI inspect Generate JSON summary of provided database files - install Install Python packages - e.g. + install Install plugins and packages from PyPI into the same... package Package specified SQLite files into a new datasette Docker... plugins List currently available plugins publish Publish specified SQLite database files to the internet along... - uninstall Uninstall Python packages (e.g. + uninstall Uninstall plugins and Python packages from the Datasette... datasette serve --help @@ -335,8 +335,7 @@ datasette install --help Usage: datasette install [OPTIONS] PACKAGES... - Install Python packages - e.g. Datasette plugins - into the same environment - as Datasette + Install plugins and packages from PyPI into the same environment as Datasette Options: -U, --upgrade Upgrade packages to latest version @@ -350,7 +349,7 @@ datasette uninstall --help Usage: datasette uninstall [OPTIONS] PACKAGES... - Uninstall Python packages (e.g. plugins) from the Datasette environment + Uninstall plugins and Python packages from the Datasette environment Options: -y, --yes Don't ask for confirmation From 8cf4b77a92f5170c33e0079f2bab48a4f36b6934 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:10:52 -0800 Subject: [PATCH 0574/1866] Better copy for 'datasette plugins --help' --- datasette/cli.py | 2 +- docs/cli-reference.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 18054448..af09453f 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -190,7 +190,7 @@ pm.hook.publish_subcommand(publish=publish) help="Path to directory containing custom plugins", ) def plugins(all, plugins_dir): - """List currently available plugins""" + """List currently installed plugins""" app = Datasette([], plugins_dir=plugins_dir) click.echo(json.dumps(app._plugins(all=all), indent=4)) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 37a30606..69a7cdd9 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -54,7 +54,7 @@ datasette --help inspect Generate JSON summary of provided database files install Install plugins and packages from PyPI into the same... package Package specified SQLite files into a new datasette Docker... - plugins List currently available plugins + plugins List currently installed plugins publish Publish specified SQLite database files to the internet along... uninstall Uninstall plugins and Python packages from the Datasette... @@ -169,7 +169,7 @@ datasette plugins --help Usage: datasette plugins [OPTIONS] - List currently available plugins + List currently installed plugins Options: --all Include built-in default plugins From 515f8d38ebae203efc15ca79a8b42848276b35e5 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:12:54 -0800 Subject: [PATCH 0575/1866] Help summaries for publish cloudrun/heroku --- datasette/publish/cloudrun.py | 1 + datasette/publish/heroku.py | 1 + docs/cli-reference.rst | 8 ++++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 1fabcafd..a1e2f580 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -74,6 +74,7 @@ def publish_subcommand(publish): cpu, apt_get_extras, ): + "Publish databases to Datasette running on Cloud Run" fail_if_publish_binary_not_installed( "gcloud", "Google Cloud", "https://cloud.google.com/sdk/" ) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 2ebbd4bd..171252ce 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -50,6 +50,7 @@ def publish_subcommand(publish): name, tar, ): + "Publish databases to Datasette running on Heroku" fail_if_publish_binary_not_installed( "heroku", "Heroku", "https://cli.heroku.com" ) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 69a7cdd9..f3279f6d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -191,8 +191,8 @@ datasette publish --help --help Show this message and exit. Commands: - cloudrun - heroku + cloudrun Publish databases to Datasette running on Cloud Run + heroku Publish databases to Datasette running on Heroku datasette publish cloudrun --help @@ -202,6 +202,8 @@ datasette publish cloudrun --help Usage: datasette publish cloudrun [OPTIONS] [FILES]... + Publish databases to Datasette running on Cloud Run + Options: -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish @@ -244,6 +246,8 @@ datasette publish heroku --help Usage: datasette publish heroku [OPTIONS] [FILES]... + Publish databases to Datasette running on Heroku + Options: -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish From 3a0f7d64889cd79d5d00d3251e8ab77ff52de60d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:27:21 -0800 Subject: [PATCH 0576/1866] Fixed hidden form fields bug #1527 --- datasette/views/table.py | 2 +- tests/test_table_html.py | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 9808fd24..77fb2850 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -815,7 +815,7 @@ class TableView(RowTableShared): if ( key.startswith("_") and key not in ("_sort", "_search", "_next") - and not key.endswith("__exact") + and "__" not in key ): for value in request.args.getlist(key): form_hidden_args.append((key, value)) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index f68e05a5..021268c3 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -143,17 +143,29 @@ def test_existing_filter_redirects(app_client): assert "?" not in response.headers["Location"] -def test_exact_parameter_results_in_correct_hidden_fields(app_client): +@pytest.mark.parametrize( + "qs,expected_hidden", + ( + # Things that should be reflected in hidden form fields: + ("_facet=_neighborhood", {"_facet": "_neighborhood"}), + ("_where=1+=+1&_col=_city_id", {"_where": "1 = 1", "_col": "_city_id"}), + # Things that should NOT be reflected in hidden form fields: + ( + "_facet=_neighborhood&_neighborhood__exact=Downtown", + {"_facet": "_neighborhood"}, + ), + ("_facet=_neighborhood&_city_id__gt=1", {"_facet": "_neighborhood"}), + ), +) +def test_reflected_hidden_form_fields(app_client, qs, expected_hidden): # https://github.com/simonw/datasette/issues/1527 - response = app_client.get( - "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown" - ) + response = app_client.get("/fixtures/facetable?{}".format(qs)) # In this case we should NOT have a hidden _neighborhood__exact=Downtown field form = Soup(response.body, "html.parser").find("form") hidden_inputs = { input["name"]: input["value"] for input in form.select("input[type=hidden]") } - assert hidden_inputs == {"_facet": "_neighborhood"} + assert hidden_inputs == expected_hidden def test_empty_search_parameter_gets_removed(app_client): From 76d66d5b2bf10249c0beaac0999b93ac8d757f48 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:30:00 -0800 Subject: [PATCH 0577/1866] Tweak order of documentation contents --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 36e42848..acca943f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -59,6 +59,6 @@ Contents plugin_hooks testing_plugins internals - contributing cli-reference + contributing changelog From 714b4df1b1b2aeab8cde3a309627c42355439dda Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:36:28 -0800 Subject: [PATCH 0578/1866] Fixed reStructuredText warning, refs #1594 --- docs/cli-reference.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f3279f6d..f529782d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -23,6 +23,7 @@ This page lists the ``--help`` for every ``datasette`` CLI command. ["install", "--help"], ["uninstall", "--help"], ] + cog.out("\n") for command in commands: title = "datasette " + " ".join(command) cog.out(title + "\n") @@ -33,6 +34,7 @@ This page lists the ``--help`` for every ``datasette`` CLI command. cog.out(textwrap.indent(output, ' ')) cog.out("\n\n") .. ]]] + datasette --help ================ From ab7d6a7179e9939c2764989e508bfa8eba31f3b1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:38:16 -0800 Subject: [PATCH 0579/1866] Updated settings help URL to avoid redirect --- datasette/cli.py | 2 +- datasette/views/base.py | 2 +- docs/cli-reference.rst | 3 ++- tests/test_html.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index af09453f..9d1b5ee5 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -400,7 +400,7 @@ def uninstall(packages, yes): "--setting", "settings", type=Setting(), - help="Setting, see docs.datasette.io/en/stable/config.html", + help="Setting, see docs.datasette.io/en/stable/settings.html", multiple=True, ) @click.option( diff --git a/datasette/views/base.py b/datasette/views/base.py index a9953dfd..b1cacb3f 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -493,7 +493,7 @@ class DataView(BaseView): raise DatasetteError( """ SQL query took too long. The time limit is controlled by the - <a href="https://docs.datasette.io/en/stable/config.html#sql-time-limit-ms">sql_time_limit_ms</a> + <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a> configuration option. """, title="SQL Interrupted", diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f529782d..74adb92d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -94,7 +94,8 @@ datasette serve --help --memory Make /_memory database available --config CONFIG Deprecated: set config option using configname:value. Use --setting instead. - --setting SETTING... Setting, see docs.datasette.io/en/stable/config.html + --setting SETTING... Setting, see + docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as signed cookies --root Output URL that sets a cookie authenticating the diff --git a/tests/test_html.py b/tests/test_html.py index bfe5c8f9..3f0a88a9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -155,7 +155,7 @@ def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)") assert 400 == response.status expected_html_fragment = """ - <a href="https://docs.datasette.io/en/stable/config.html#sql-time-limit-ms">sql_time_limit_ms</a> + <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a> """.strip() assert expected_html_fragment in response.text From 10659c3f1f82458adfa65c61f4dcc8d9af5467ed Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:38:53 -0800 Subject: [PATCH 0580/1866] datasette-debug-asgi plugin to help investigate #1590 --- demos/apache-proxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index ab7b9d16..6c921963 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -27,7 +27,7 @@ ARG DATASETTE_REF RUN pip install \ https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip \ - datasette-redirect-to-https + datasette-redirect-to-https datasette-debug-asgi ADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf From 3664ddd400062123e99500d28b160c7944408c1a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 16:47:53 -0800 Subject: [PATCH 0581/1866] Replace update-docs-help.py with cog, closes #1598 --- docs/cli-reference.rst | 24 ++++++++++++++ docs/datasette-package-help.txt | 29 ---------------- docs/datasette-publish-cloudrun-help.txt | 33 ------------------- docs/datasette-publish-heroku-help.txt | 29 ---------------- docs/datasette-serve-help.txt | 42 ------------------------ docs/getting_started.rst | 9 +---- docs/publish.rst | 6 ++-- tests/test_docs.py | 20 ----------- update-docs-help.py | 25 -------------- 9 files changed, 28 insertions(+), 189 deletions(-) delete mode 100644 docs/datasette-package-help.txt delete mode 100644 docs/datasette-publish-cloudrun-help.txt delete mode 100644 docs/datasette-publish-heroku-help.txt delete mode 100644 docs/datasette-serve-help.txt delete mode 100644 update-docs-help.py diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 74adb92d..155a005d 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -26,6 +26,8 @@ This page lists the ``--help`` for every ``datasette`` CLI command. cog.out("\n") for command in commands: title = "datasette " + " ".join(command) + ref = "_cli_help_" + ("_".join(command).replace("-", "_")) + cog.out(".. {}:\n\n".format(ref)) cog.out(title + "\n") cog.out(("=" * len(title)) + "\n\n") cog.out("::\n\n") @@ -35,6 +37,8 @@ This page lists the ``--help`` for every ``datasette`` CLI command. cog.out("\n\n") .. ]]] +.. _cli_help___help: + datasette --help ================ @@ -61,6 +65,8 @@ datasette --help uninstall Uninstall plugins and Python packages from the Datasette... +.. _cli_help_serve___help: + datasette serve --help ====================== @@ -114,6 +120,8 @@ datasette serve --help --help Show this message and exit. +.. _cli_help_serve___help_settings: + datasette serve --help-settings =============================== @@ -165,6 +173,8 @@ datasette serve --help-settings +.. _cli_help_plugins___help: + datasette plugins --help ======================== @@ -180,6 +190,8 @@ datasette plugins --help --help Show this message and exit. +.. _cli_help_publish___help: + datasette publish --help ======================== @@ -198,6 +210,8 @@ datasette publish --help heroku Publish databases to Datasette running on Heroku +.. _cli_help_publish_cloudrun___help: + datasette publish cloudrun --help ================================= @@ -242,6 +256,8 @@ datasette publish cloudrun --help --help Show this message and exit. +.. _cli_help_publish_heroku___help: + datasette publish heroku --help =============================== @@ -281,6 +297,8 @@ datasette publish heroku --help --help Show this message and exit. +.. _cli_help_package___help: + datasette package --help ======================== @@ -317,6 +335,8 @@ datasette package --help --help Show this message and exit. +.. _cli_help_inspect___help: + datasette inspect --help ======================== @@ -335,6 +355,8 @@ datasette inspect --help --help Show this message and exit. +.. _cli_help_install___help: + datasette install --help ======================== @@ -349,6 +371,8 @@ datasette install --help --help Show this message and exit. +.. _cli_help_uninstall___help: + datasette uninstall --help ========================== diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt deleted file mode 100644 index 7cfac1b1..00000000 --- a/docs/datasette-package-help.txt +++ /dev/null @@ -1,29 +0,0 @@ -$ datasette package --help - -Usage: datasette package [OPTIONS] FILES... - - Package specified SQLite files into a new datasette Docker container - -Options: - -t, --tag TEXT Name for the resulting Docker container, can optionally use - name:tag format - -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. main - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --install TEXT Additional packages (e.g. plugins) to install - --spatialite Enable SpatialLite extension - --version-note TEXT Additional note to show on /-/versions - --secret TEXT Secret used for signing secure values, such as signed - cookies - -p, --port INTEGER RANGE Port to run the server on, defaults to 8001 [1<=x<=65535] - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - --help Show this message and exit. diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt deleted file mode 100644 index 34481b40..00000000 --- a/docs/datasette-publish-cloudrun-help.txt +++ /dev/null @@ -1,33 +0,0 @@ -$ datasette publish cloudrun --help - -Usage: datasette publish cloudrun [OPTIONS] [FILES]... - -Options: - -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. main - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --install TEXT Additional packages (e.g. plugins) to install - --plugin-secret <TEXT TEXT TEXT>... - Secrets to pass to plugins, e.g. --plugin-secret - datasette-auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions - --secret TEXT Secret used for signing secure values, such as signed - cookies - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when building - --service TEXT Cloud Run service to deploy (or over-write) - --spatialite Enable SpatialLite extension - --show-files Output the generated Dockerfile and metadata.json - --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi - --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run - --apt-get-install TEXT Additional packages to apt-get install - --help Show this message and exit. diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt deleted file mode 100644 index 9d633e95..00000000 --- a/docs/datasette-publish-heroku-help.txt +++ /dev/null @@ -1,29 +0,0 @@ -$ datasette publish heroku --help - -Usage: datasette publish heroku [OPTIONS] [FILES]... - -Options: - -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish - --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. main - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --install TEXT Additional packages (e.g. plugins) to install - --plugin-secret <TEXT TEXT TEXT>... - Secrets to pass to plugins, e.g. --plugin-secret - datasette-auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions - --secret TEXT Secret used for signing secure values, such as signed - cookies - --title TEXT Title for metadata - --license TEXT License label for metadata - --license_url TEXT License URL for metadata - --source TEXT Source label for metadata - --source_url TEXT Source URL for metadata - --about TEXT About label for metadata - --about_url TEXT About URL for metadata - -n, --name TEXT Application name to use when deploying - --tar TEXT --tar option to pass to Heroku, e.g. - --tar=/usr/local/bin/gtar - --help Show this message and exit. diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt deleted file mode 100644 index 2911977a..00000000 --- a/docs/datasette-serve-help.txt +++ /dev/null @@ -1,42 +0,0 @@ -$ datasette serve --help - -Usage: datasette serve [OPTIONS] [FILES]... - - Serve up specified SQLite database files with a web UI - -Options: - -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means only - connections from the local machine will be allowed. Use - 0.0.0.0 to listen to all IPs and allow access from other - machines. - -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to automatically - assign an available port. [0<=x<=65535] - --uds TEXT Bind to a Unix domain socket - --reload Automatically reload if code or metadata change detected - - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: * - --load-extension TEXT Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON/YAML file containing license/source metadata - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --memory Make /_memory database available - --config CONFIG Deprecated: set config option using configname:value. Use - --setting instead. - --setting SETTING... Setting, see docs.datasette.io/en/stable/config.html - --secret TEXT Secret used for signing secure values, such as signed - cookies - --root Output URL that sets a cookie authenticating the root user - --get TEXT Run an HTTP GET request against this path, print results and - exit - --version-note TEXT Additional note to show on /-/versions - --help-settings Show available settings - --pdb Launch debugger on any errors - -o, --open Open Datasette in your web browser - --create Create database files if they do not exist - --crossdb Enable cross-database joins using the /_memory database - --ssl-keyfile TEXT SSL key file - --ssl-certfile TEXT SSL certificate file - --help Show this message and exit. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 52434fdc..3e357afb 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -161,11 +161,4 @@ The ``--get`` option can specify the path to a page within Datasette and cause D The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. -.. _getting_started_serve_help: - -datasette serve --help ----------------------- - -Running ``datasette downloads.db`` executes the default ``serve`` sub-command, and is equivalent to running ``datasette serve downloads.db``. The full list of options to that command is shown below. - -.. literalinclude:: datasette-serve-help.txt +Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. diff --git a/docs/publish.rst b/docs/publish.rst index f6895f53..1d9664e7 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -47,7 +47,7 @@ Once it has finished it will output a URL like this one:: Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your own domain or subdomain at your Cloud Run service - see `mapping custom domains <https://cloud.google.com/run/docs/mapping-custom-domains>`__ in the Cloud Run documentation for details. -.. literalinclude:: datasette-publish-cloudrun-help.txt +See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command. Publishing to Heroku -------------------- @@ -64,7 +64,7 @@ This will output some details about the new deployment, including a URL like thi You can specify a custom app name by passing ``-n my-app-name`` to the publish command. This will also allow you to overwrite an existing app. -.. literalinclude:: datasette-publish-heroku-help.txt +See :ref:`cli_help_publish_heroku___help` for the full list of options for this command. .. _publish_vercel: @@ -171,4 +171,4 @@ You can customize the port that is exposed by the container using the ``--port`` A full list of options can be seen by running ``datasette package --help``: -.. literalinclude:: datasette-package-help.txt +See :ref:`cli_help_package___help` for the full list of options for this command. \ No newline at end of file diff --git a/tests/test_docs.py b/tests/test_docs.py index d0cb036d..0d17b8e3 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -33,26 +33,6 @@ def test_settings_are_documented(settings_headings, setting): assert setting.name in settings_headings -@pytest.mark.parametrize( - "name,filename", - ( - ("serve", "datasette-serve-help.txt"), - ("package", "datasette-package-help.txt"), - ("publish heroku", "datasette-publish-heroku-help.txt"), - ("publish cloudrun", "datasette-publish-cloudrun-help.txt"), - ), -) -def test_help_includes(name, filename): - expected = (docs_path / filename).read_text() - runner = CliRunner() - result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) - actual = f"$ datasette {name} --help\n\n{result.output}" - # actual has "Usage: cli package [OPTIONS] FILES" - # because it doesn't know that cli will be aliased to datasette - expected = expected.replace("Usage: datasette", "Usage: cli") - assert expected == actual, "Run python update-docs-help.py to fix this" - - @pytest.fixture(scope="session") def plugin_hooks_content(): return (docs_path / "plugin_hooks.rst").read_text() diff --git a/update-docs-help.py b/update-docs-help.py deleted file mode 100644 index 292d1dcd..00000000 --- a/update-docs-help.py +++ /dev/null @@ -1,25 +0,0 @@ -from click.testing import CliRunner -from datasette.cli import cli -from pathlib import Path - -docs_path = Path(__file__).parent / "docs" - -includes = ( - ("serve", "datasette-serve-help.txt"), - ("package", "datasette-package-help.txt"), - ("publish heroku", "datasette-publish-heroku-help.txt"), - ("publish cloudrun", "datasette-publish-cloudrun-help.txt"), -) - - -def update_help_includes(): - for name, filename in includes: - runner = CliRunner() - result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) - actual = f"$ datasette {name} --help\n\n{result.output}" - actual = actual.replace("Usage: cli ", "Usage: datasette ") - (docs_path / filename).write_text(actual) - - -if __name__ == "__main__": - update_help_includes() From cb29119db9115b1f40de2fb45263ed77e3bfbb3e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Jan 2022 17:36:51 -0800 Subject: [PATCH 0582/1866] Release 0.60 Refs #473, #625, #1527, #1544, #1547, #1551, #1552, #1555, #1556, #1557, #1563, #1564, #1568, #1570, #1575, #1579, #1588, #1594 --- datasette/version.py | 2 +- docs/changelog.rst | 37 ++++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 290fbcf3..a4e340b3 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.60a1" +__version__ = "0.60" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99d3315e..d7e2af39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,30 +4,41 @@ Changelog ========= -.. _v0_60a1: +.. _v0_60: -0.60a1 (2021-12-19) -------------------- +0.60 (2022-01-13) +----------------- +Plugins and internals +~~~~~~~~~~~~~~~~~~~~~ + +- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`) +- Added two additional methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`) +- The :ref:`db.execute_write() <database_execute_write>` internal method now defaults to blocking until the write operation has completed. Previously it defaulted to queuing the write and then continuing to run code while the write was in the queue. (:issue:`1579`) - Database write connections now execute the :ref:`plugin_hook_prepare_connection` plugin hook. (:issue:`1564`) - The ``Datasette()`` constructor no longer requires the ``files=`` argument, and is now documented at :ref:`internals_datasette`. (:issue:`1563`) - The tracing feature now traces write queries, not just read queries. (:issue:`1568`) -- Added two methods for writing to the database: :ref:`database_execute_write_script` and :ref:`database_execute_write_many`. (:issue:`1570`) -- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`) -- Fixed bug where writable canned queries could not be used with custom templates. (:issue:`1547`) +- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`) -.. _v0_60a0: +Faceting +~~~~~~~~ -0.60a0 (2021-12-17) -------------------- - -- New plugin hook: :ref:`plugin_hook_filters_from_request`, which runs on the table page and can be used to support new custom query string parameters that modify the SQL query. (:issue:`473`) - The number of unique values in a facet is now always displayed. Previously it was only displayed if the user specified ``?_facet_size=max``. (:issue:`1556`) -- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`) - Facets of type ``date`` or ``array`` can now be configured in ``metadata.json``, see :ref:`facets_metadata`. Thanks, David Larlet. (:issue:`1552`) - New ``?_nosuggest=1`` parameter for table views, which disables facet suggestion. (:issue:`1557`) +- Fixed bug where ``?_facet_array=tags&_facet=tags`` would only display one of the two selected facets. (:issue:`625`) + +Other small fixes +~~~~~~~~~~~~~~~~~ + +- Made several performance improvements to the database schema introspection code that runs when Datasette first starts up. (:issue:`1555`) - Label columns detected for foreign keys are now case-insensitive, so ``Name`` or ``TITLE`` will be detected in the same way as ``name`` or ``title``. (:issue:`1544`) -- The query string variables exposed by ``request.args`` will now include blank strings for arguments such as ``foo`` in ``?foo=&bar=1`` rather than ignoring those parameters entirely. (:issue:`1551`) +- Upgraded Pluggy dependency to 1.0. (:issue:`1575`) +- Now using `Plausible analytics <https://plausible.io/>`__ for the Datasette documentation. +- ``explain query plan`` is now allowed with varying amounts of whitespace in the query. (:issue:`1588`) +- New :ref:`cli_reference` page showing the output of ``--help`` for each of the ``datasette`` sub-commands. This lead to several small improvements to the help copy. (:issue:`1594`) +- Fixed bug where writable canned queries could not be used with custom templates. (:issue:`1547`) +- Improved fix for a bug where columns with a underscore prefix could result in unnecessary hidden form fields. (:issue:`1527`) .. _v0_59_4: From 58652dd925bb7509b43905423ec00083bd374dc1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 20:12:46 -0800 Subject: [PATCH 0583/1866] Hidden tables sqlite1/2/3/4, closes #1587 --- datasette/database.py | 4 +++- tests/test_api.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index e908d1ea..06dc8da5 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -345,7 +345,9 @@ class Database: """ select name from sqlite_master where rootpage = 0 - and sql like '%VIRTUAL TABLE%USING FTS%' + and ( + sql like '%VIRTUAL TABLE%USING FTS%' + ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') """ ) ).rows diff --git a/tests/test_api.py b/tests/test_api.py index 574ebb41..47ec3a8c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1030,3 +1030,15 @@ async def test_db_path(app_client): # Previously this broke if path was a pathlib.Path: await datasette.refresh_schemas() + + +@pytest.mark.asyncio +async def test_hidden_sqlite_stat1_table(): + ds = Datasette() + db = ds.add_memory_database("db") + await db.execute_write("create table normal (id integer primary key, name text)") + await db.execute_write("create index idx on normal (name)") + await db.execute_write("analyze") + data = (await ds.client.get("/db.json?_show_hidden=1")).json() + tables = [(t["name"], t["hidden"]) for t in data["tables"]] + assert tables == [("normal", False), ("sqlite_stat1", True)] From fae3983c51f4a3aca8335f3e01ff85ef27076fbf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 20:31:22 -0800 Subject: [PATCH 0584/1866] Drop support for Python 3.6, closes #1577 Refs #1606 --- .github/workflows/publish.yml | 6 +++--- .github/workflows/test.yml | 2 +- README.md | 2 +- docs/contributing.rst | 2 +- docs/installation.rst | 2 +- docs/introspection.rst | 8 ++++---- setup.py | 3 +-- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 17c6ae9b..3cfc67da 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -38,7 +38,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/cache@v2 name: Configure pip caching with: @@ -66,7 +66,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.10' - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 704931a6..78c289bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index ce15ccf4..107d81da 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.7 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/docs/contributing.rst b/docs/contributing.rst index 07f2a0e4..b74f2f36 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -19,7 +19,7 @@ General guidelines Setting up a development environment ------------------------------------ -If you have Python 3.6 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew <https://docs.python-guide.org/starting/install3/osx/>`__) you can install an editable copy of Datasette using the following steps. +If you have Python 3.7 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew <https://docs.python-guide.org/starting/install3/osx/>`__) you can install an editable copy of Datasette using the following steps. If you want to use GitHub to publish your changes, first `create a fork of datasette <https://github.com/simonw/datasette/fork>`__ under your own GitHub account. diff --git a/docs/installation.rst b/docs/installation.rst index ac3dcca2..e8bef9cd 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -56,7 +56,7 @@ If the latest packaged release of Datasette has not yet been made available thro Using pip --------- -Datasette requires Python 3.6 or higher. Visit `InstallPython3.com <https://installpython3.com/>`__ for step-by-step installation guides for your operating system. +Datasette requires Python 3.7 or higher. Visit `InstallPython3.com <https://installpython3.com/>`__ for step-by-step installation guides for your operating system. You can install Datasette and its dependencies using ``pip``:: diff --git a/docs/introspection.rst b/docs/introspection.rst index d1a0a854..e08ca911 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -38,11 +38,11 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la { "datasette": { - "version": "0.21" + "version": "0.60" }, "python": { - "full": "3.6.5 (default, May 5 2018, 03:07:21) \n[GCC 6.3.0 20170516]", - "version": "3.6.5" + "full": "3.8.12 (default, Dec 21 2021, 10:45:09) \n[GCC 10.2.1 20210110]", + "version": "3.8.12" }, "sqlite": { "extensions": { @@ -62,7 +62,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la "ENABLE_RTREE", "THREADSAFE=1" ], - "version": "3.16.2" + "version": "3.37.0" } } diff --git a/setup.py b/setup.py index e9ef082a..dade0a88 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( packages=find_packages(exclude=("tests",)), package_data={"datasette": ["templates/*.html"]}, include_package_data=True, - python_requires=">=3.6", + python_requires=">=3.7", install_requires=[ "asgiref>=3.2.10,<3.5.0", "click>=7.1.1,<8.1.0", @@ -91,6 +91,5 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.6", ], ) From 14e320329f756b7d8e298c4e2251d8a0b194c9c4 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 20:38:49 -0800 Subject: [PATCH 0585/1866] Hidden tables data_licenses, KNN, KNN2 for SpatiaLite, closes #1601 --- datasette/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/database.py b/datasette/database.py index 06dc8da5..6ce87215 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -365,6 +365,9 @@ class Database: "sqlite_sequence", "views_geometry_columns", "virts_geometry_columns", + "data_licenses", + "KNN", + "KNN2", ] + [ r[0] for r in ( From 43c30ce0236ebbc7e9cec98a3822265eb2691430 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:04:09 -0800 Subject: [PATCH 0586/1866] Use cog to maintain default plugin list in plugins.rst, closes #1600 Also fixed a bug I spotted where datasette.filters showed the same hook three times. --- datasette/app.py | 2 +- docs/plugins.rst | 97 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bd663509..0a89a9f3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -770,7 +770,7 @@ class Datasette: "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": p["hooks"], + "hooks": list(set(p["hooks"])), } for p in ps ] diff --git a/docs/plugins.rst b/docs/plugins.rst index 020030f1..4a2c0194 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -91,36 +91,119 @@ You can also use the ``datasette plugins`` command:: } ] +.. [[[cog + from datasette import cli + from click.testing import CliRunner + import textwrap, json + cog.out("\n") + result = CliRunner().invoke(cli.cli, ["plugins", "--all"]) + # cog.out() with text containing newlines was unindenting for some reason + cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::\n") + plugins = [p for p in json.loads(result.output) if p["name"].startswith("datasette.")] + indented = textwrap.indent(json.dumps(plugins, indent=4), " ") + for line in indented.split("\n"): + cog.outl(line) + cog.out("\n\n") +.. ]]] + If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: - $ datasette plugins --all [ + { + "name": "datasette.publish.heroku", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "publish_subcommand" + ] + }, { "name": "datasette.sql_functions", "static": false, "templates": false, - "version": null + "version": null, + "hooks": [ + "prepare_connection" + ] }, { - "name": "datasette.publish.cloudrun", + "name": "datasette.actor_auth_cookie", "static": false, "templates": false, - "version": null + "version": null, + "hooks": [ + "actor_from_request" + ] + }, + { + "name": "datasette.blob_renderer", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "register_output_renderer" + ] }, { "name": "datasette.facets", "static": false, "templates": false, - "version": null + "version": null, + "hooks": [ + "register_facet_classes" + ] }, { - "name": "datasette.publish.heroku", + "name": "datasette.default_magic_parameters", "static": false, "templates": false, - "version": null + "version": null, + "hooks": [ + "register_magic_parameters" + ] + }, + { + "name": "datasette.default_permissions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "permission_allowed" + ] + }, + { + "name": "datasette.default_menu_links", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "menu_links" + ] + }, + { + "name": "datasette.filters", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "filters_from_request" + ] + }, + { + "name": "datasette.publish.cloudrun", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "publish_subcommand" + ] } ] + +.. [[[end]]] + You can add the ``--plugins-dir=`` option to include any plugins found in that directory. .. _plugins_configuration: From e1770766ce3ae6669305662ba618be610367af77 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:14:04 -0800 Subject: [PATCH 0587/1866] Return plugins and hooks in predictable order --- datasette/app.py | 3 ++- docs/plugins.rst | 58 +++++++++++++++++++++++------------------------ tests/test_cli.py | 4 +--- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0a89a9f3..49858a4a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -764,13 +764,14 @@ class Datasette: should_show_all = all if not should_show_all: ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] + ps.sort(key=lambda p: p["name"]) return [ { "name": p["name"], "static": p["static_path"] is not None, "templates": p["templates_path"] is not None, "version": p.get("version"), - "hooks": list(set(p["hooks"])), + "hooks": list(sorted(set(p["hooks"]))), } for p in ps ] diff --git a/docs/plugins.rst b/docs/plugins.rst index 4a2c0194..f2ed02f7 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -109,24 +109,6 @@ You can also use the ``datasette plugins`` command:: If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: [ - { - "name": "datasette.publish.heroku", - "static": false, - "templates": false, - "version": null, - "hooks": [ - "publish_subcommand" - ] - }, - { - "name": "datasette.sql_functions", - "static": false, - "templates": false, - "version": null, - "hooks": [ - "prepare_connection" - ] - }, { "name": "datasette.actor_auth_cookie", "static": false, @@ -145,15 +127,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_output_renderer" ] }, - { - "name": "datasette.facets", - "static": false, - "templates": false, - "version": null, - "hooks": [ - "register_facet_classes" - ] - }, { "name": "datasette.default_magic_parameters", "static": false, @@ -163,6 +136,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_magic_parameters" ] }, + { + "name": "datasette.default_menu_links", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "menu_links" + ] + }, { "name": "datasette.default_permissions", "static": false, @@ -173,12 +155,12 @@ If you run ``datasette plugins --all`` it will include default plugins that ship ] }, { - "name": "datasette.default_menu_links", + "name": "datasette.facets", "static": false, "templates": false, "version": null, "hooks": [ - "menu_links" + "register_facet_classes" ] }, { @@ -198,6 +180,24 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "hooks": [ "publish_subcommand" ] + }, + { + "name": "datasette.publish.heroku", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "publish_subcommand" + ] + }, + { + "name": "datasette.sql_functions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "prepare_connection" + ] } ] diff --git a/tests/test_cli.py b/tests/test_cli.py index 763fe2e7..bbc5df30 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -106,9 +106,7 @@ def test_spatialite_error_if_cannot_find_load_extension_spatialite(): def test_plugins_cli(app_client): runner = CliRunner() result1 = runner.invoke(cli, ["plugins"]) - assert sorted(EXPECTED_PLUGINS, key=lambda p: p["name"]) == sorted( - json.loads(result1.output), key=lambda p: p["name"] - ) + assert json.loads(result1.output) == EXPECTED_PLUGINS # Try with --all result2 = runner.invoke(cli, ["plugins", "--all"]) names = [p["name"] for p in json.loads(result2.output)] From 0467723ee57c2cbc0f02daa47cef632dd4651df0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:46:03 -0800 Subject: [PATCH 0588/1866] New, improved favicon - refs #1603 --- datasette/app.py | 11 ++++++++++- datasette/static/favicon.png | Bin 0 -> 1207 bytes tests/test_html.py | 4 +++- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 datasette/static/favicon.png diff --git a/datasette/app.py b/datasette/app.py index 49858a4a..b2942cd9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -70,6 +70,7 @@ from .utils.asgi import ( Response, asgi_static, asgi_send, + asgi_send_file, asgi_send_html, asgi_send_json, asgi_send_redirect, @@ -178,9 +179,17 @@ SETTINGS = ( DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} +FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" + async def favicon(request, send): - await asgi_send(send, "", 200) + await asgi_send_file( + send, + str(FAVICON_PATH), + content_type="image/png", + chunk_size=4096, + headers={"Cache-Control": "max-age=3600, immutable, public"}, + ) class Datasette: diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..36d6334fd7714da87aad44bba0d9c2a869cd0b86 GIT binary patch literal 1207 zcmeAS@N?(olHy`uVBq!ia0vp^86eET1|(%=wk`xxoCO|{#S9GG!XV7ZFl&wkP>``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eBu8*gSV@L(#+gXPB%B~{E@6V2KU$RE|a=`mo z=TfEn3fnR!zH@Oseo)=wmh*$NvI}i5UR0HpZD-+iO=*9nWijR7qiH4)<$WST)$;i@ z7Pn&6W#%ORmVfzw^0n8MkLM=U{Z2c!d+{f!Nh;ZFoU_%gg<K8^2~y1O^E`j0N^4WS z)t0-HC$IUQZ-4!?)*=bf+}{^PjLiQWdq0o&?c~!gYhy%W{N?P*4DbJ%$Td@YV}E$l zz53T~-v6xLzVW{<o?)`-SjGzf8K$QX&1;>t_1E7u_MKO{uKo<L)=N)*Wi*l5{#(e? z19Rm#E0fQ>nKnPKDs9jE<8O=B=Jm9FI<kL@+QiR_=i0Y!`~P6ESF!uD#ORRy!B?Hv z+lv_q-|MgM`yEtze)7W$`G3k|LyF?`XZ<R;{^8-f=i#Z#H_lpGsVg%rded{MCyw*j zy=H2E>zh^cE<R6m@y;h}4=TKFi;%clvb^Zx#Lm3nb4nMKr+zqK{IWmjO8AAxpIdJV z=iJt@5#8jGxBBMwqYh^lO}PFg&d?%GAmE%)2lvM2B!wSYRuu)CCBArO`k$C|At=ek zJn_$uWe$4ZVonqVR7dG$Nv%46EkygXV5Gv-|1;jHaWu9y9atI4yzk2IKn|WW3I!S* zGF(jQS$tK7KY~)-UVU|!7dfpTU-xnHnG>PgJ8#X~AG#rFrX&j=YooK{g8%^w5yH&A zu=@`Lb!Nbnmw#>8dOX(lIu939d((l1P%H5<cVx22q=BsR^F6U?rJvzaTg5vjOxNVh zdt;2j?!~7WqCo#m+JXCb=j^xMe5!j}(p{Z`%N!Omk3hCtu>Z9eGn54i6jk3bDLT6V z(?LnOy0g1j|L5P)W(y179CBjQ4UMUHd=0k78I~oh-Vxm-($q3(!3(t?#dH69edwFe z8oYk{U(GqSF+Hy4isG-QHQ(}?v_>-}(Im|H|CQYv-*<n!yzp%sFs`FqgEnlqoRPw- zSbY4q<-EKsIXUsi@0Ju7P5OGszIE-TP_0b6D}3TSpH)3?ef&}uclmwteDT-k+P-Dl z^s^q-De}EPMgKZ)yg|vUuj?(;V-@QzpZ;}Ad8K-^-O&tfd+z<m_UZjUwJqt}k^JZH zo8x$HegC`YyVv6d(V?g7KU9S4ym@By>+cVXYPIVVUa9r1zW3x~{B3L5XDjDF@~uA} zp6fgF*VMny^fq~}43T|4*J6vWv}(7)Icsgx-y80q|26qW>$K~W*PdB@%So*DsOB-_ z=bN*RF3o)Fy}u;n^4wRK`eQW3!Y|a>)NJLO@oekkJdvlXd#cw}1p8iomBlP^=5O7< pA8K}rANu~Q{$t<kbEaiK<ImRe6uW@kXMyDmgQu&X%Q~loCIC6tDUtvH literal 0 HcmV?d00001 diff --git a/tests/test_html.py b/tests/test_html.py index 3f0a88a9..735d12ff 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -63,7 +63,9 @@ def test_homepage_options(app_client): def test_favicon(app_client): response = app_client.get("/favicon.ico") assert response.status == 200 - assert "" == response.text + assert response.headers["cache-control"] == "max-age=3600, immutable, public" + assert response.headers["content-length"] == "1207" + assert response.headers["content-type"] == "image/png" def test_static(app_client): From b2eebf5ebf222b61a21625527851b77347d3d662 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:52:00 -0800 Subject: [PATCH 0589/1866] No need to send this, it's got a default, refs #1603 --- datasette/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index b2942cd9..09d7d034 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -187,7 +187,6 @@ async def favicon(request, send): send, str(FAVICON_PATH), content_type="image/png", - chunk_size=4096, headers={"Cache-Control": "max-age=3600, immutable, public"}, ) From b01c9b68d151e1656fc180815c0d8480e35fc961 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:54:41 -0800 Subject: [PATCH 0590/1866] Oops I pushed the wrong favicon, refs #1603 --- datasette/static/favicon.png | Bin 1207 -> 1358 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png index 36d6334fd7714da87aad44bba0d9c2a869cd0b86..a27139ae6ecb09660f96d262d3779de29f7b5d4f 100644 GIT binary patch literal 1358 zcmV-U1+n^xP)<h;3K|Lk000e1NJLTq004jh004jp0{{R3^x%>C0000;P)t-s0001% zmb`Fqr$j`OPfwIOI*uG1hYt>g|NsB|{QdCp_1@p^(bD6=!P>XE+_AIRsjA42k-T+v zrB+v&Dk+Hr0*3$qfB*n>9JfE)00008bW%=J07QSN(*FMbUcFRk000E7Nkl<Zc-rk- zje4sf486525v_s>@%`WQlKxzNVr%CWzi#uW>voplSwP4@{_OZL9ExLB6vg4y?TeGK zxnWMl)7zgdqJ`^`=cF%a;XEnfZ~_DX0hkjY04$p0#Qg^g4mhGP`9S3ZfVHOsyFUU7 z#G%tFCv-O;3Iy9H0!|1B<DlF8=+1O68`nZ%>HhuALbX|-?j<*B7==YsgcaaKiLv)L zR~e4ruNx2mhbRC<c#EtdAW$ucAO!R?YO?W`YeNO!G#vuildl268^F5pk3*oYm4N2p zYrq0_83Vw}>_!pP!O;Q=uhM_!kwFpBR0sbAoGjp31uE6rmB2!vqppAi3=e@S_*n0v zGty(YfGe8&)i!jlbNTZ8k6gf6ZwrwM*c>(yU^fP)5dafI5T+Q;E%LO90Kjouuqqgu z7@dQT7){iyRiobWOrHWV_GJXwv`t$_RNXqRjB#<?CGl(>SH)J`Q=U;o0<f1M%8L4n zx(vCRdXy@nWaF$ZW$dZi#Iw9Ot0pO*3t)c-6%*nrK*nY_0TC+2JZ}O*W>$dY0_27Q z$RHIUH;sTSKQ96sw}B@SP)h{JpA><;4XoA%KC&m=ek@?4VT8>`!{HJ`ce-oqfT)#Y zG_zt{LtImb)Rd<|@j4?y#CNGBW;`PT5d#3;r52f4oE0EAMGWT_dBSW`iX%=j3pmE` zD&UQ<=^Ru<DWcLw=qNqm+0I&-$;VLAHS-vHBU#G(V<>&E_v$f}|CN-MwPV)mi1yKN z&k6T7@C`De4~@XjZUetOC*%{AUGI*Bany_z!BPFZm06OIaB4as!7nEye4bD&o+lK` z<_Xc_#$W<qTUbnU!YV?ol+u@ox@gB$F<Py3lXz<8**GheSWP~LlJ|r%?T9=@pN+Gk zoG|Ues(+lkbwoZVeBKe2RJgpIkDidoJNFRqps2h@N93)1M|3~@VC2i+bHe|~2Y!77 z+Fzd$?QI~d067%_`Fg!TJ{KUpBNrgQK^u5?rnKb(5>G!Ey%GRu`WS=5JJd8cZ<Q`y zTjVo)lhgt;ob<GBtR26(PQ1v|@(2XSU3SEICgGNNj>sZlW@~p?wG9Bmwok!=u6XmF za9=zA4x{0o59~SN-Uc%Fgx*9x%?T?{h5sFgo{D*{mJ_}|7*$_P$@mOOwek#!nw%k- z&xkn1IkU*qe%uR<%OW5Mi#%ceeL~O7P>ama&b-vxJEU{P4q7v=TjVJa)|f4CwtJt@ zPtyg|m(~l)4}YI<kkdJ#XVw*9*n&m1?PwUymLDwu)}_p!5BTwE6c&nyfC8);n^JBT z0a@zN1UoDa2B{Hjm=jh80nfoUCkui?kUs)G0MS;bf&@`3$lV5f3jj2Sc&C@32SLj0 z5pbV1B2WO*tyXBRWj$LtUF9W~t7=36I8XpQ3PB3Su9lfsPUWQ^5qt$i3P6n5cPe;C zzD?heS51hvBaoGc=AIql{>xz;1prQLx-0;}3cxWlgs^^}S-R&EEeN*$DIOO*C~-R< zX5L0jdwPaSLA1l+>8@9c&tY)|*l^D)M{zuxv&jcn`mvbX`kWVkc6=QE0tPO=y`~{k QD*ylh07*qoM6N<$g1B&Oe*gdg literal 1207 zcmeAS@N?(olHy`uVBq!ia0vp^86eET1|(%=wk`xxoCO|{#S9GG!XV7ZFl&wkP>``W z$lZxy-8q?;Kn_c~qpu?a!^VE@KZ&eBu8*gSV@L(#+gXPB%B~{E@6V2KU$RE|a=`mo z=TfEn3fnR!zH@Oseo)=wmh*$NvI}i5UR0HpZD-+iO=*9nWijR7qiH4)<$WST)$;i@ z7Pn&6W#%ORmVfzw^0n8MkLM=U{Z2c!d+{f!Nh;ZFoU_%gg<K8^2~y1O^E`j0N^4WS z)t0-HC$IUQZ-4!?)*=bf+}{^PjLiQWdq0o&?c~!gYhy%W{N?P*4DbJ%$Td@YV}E$l zz53T~-v6xLzVW{<o?)`-SjGzf8K$QX&1;>t_1E7u_MKO{uKo<L)=N)*Wi*l5{#(e? z19Rm#E0fQ>nKnPKDs9jE<8O=B=Jm9FI<kL@+QiR_=i0Y!`~P6ESF!uD#ORRy!B?Hv z+lv_q-|MgM`yEtze)7W$`G3k|LyF?`XZ<R;{^8-f=i#Z#H_lpGsVg%rded{MCyw*j zy=H2E>zh^cE<R6m@y;h}4=TKFi;%clvb^Zx#Lm3nb4nMKr+zqK{IWmjO8AAxpIdJV z=iJt@5#8jGxBBMwqYh^lO}PFg&d?%GAmE%)2lvM2B!wSYRuu)CCBArO`k$C|At=ek zJn_$uWe$4ZVonqVR7dG$Nv%46EkygXV5Gv-|1;jHaWu9y9atI4yzk2IKn|WW3I!S* zGF(jQS$tK7KY~)-UVU|!7dfpTU-xnHnG>PgJ8#X~AG#rFrX&j=YooK{g8%^w5yH&A zu=@`Lb!Nbnmw#>8dOX(lIu939d((l1P%H5<cVx22q=BsR^F6U?rJvzaTg5vjOxNVh zdt;2j?!~7WqCo#m+JXCb=j^xMe5!j}(p{Z`%N!Omk3hCtu>Z9eGn54i6jk3bDLT6V z(?LnOy0g1j|L5P)W(y179CBjQ4UMUHd=0k78I~oh-Vxm-($q3(!3(t?#dH69edwFe z8oYk{U(GqSF+Hy4isG-QHQ(}?v_>-}(Im|H|CQYv-*<n!yzp%sFs`FqgEnlqoRPw- zSbY4q<-EKsIXUsi@0Ju7P5OGszIE-TP_0b6D}3TSpH)3?ef&}uclmwteDT-k+P-Dl z^s^q-De}EPMgKZ)yg|vUuj?(;V-@QzpZ;}Ad8K-^-O&tfd+z<m_UZjUwJqt}k^JZH zo8x$HegC`YyVv6d(V?g7KU9S4ym@By>+cVXYPIVVUa9r1zW3x~{B3L5XDjDF@~uA} zp6fgF*VMny^fq~}43T|4*J6vWv}(7)Icsgx-y80q|26qW>$K~W*PdB@%So*DsOB-_ z=bN*RF3o)Fy}u;n^4wRK`eQW3!Y|a>)NJLO@oekkJdvlXd#cw}1p8iomBlP^=5O7< pA8K}rANu~Q{$t<kbEaiK<ImRe6uW@kXMyDmgQu&X%Q~loCIC6tDUtvH From 7c67483f5e61f7c46410a433a55280d62280327f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 19 Jan 2022 21:57:14 -0800 Subject: [PATCH 0591/1866] Make test_favicon flexible to changing icon sizes, refs #1603 --- tests/test_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_html.py b/tests/test_html.py index 735d12ff..aa718857 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -64,7 +64,7 @@ def test_favicon(app_client): response = app_client.get("/favicon.ico") assert response.status == 200 assert response.headers["cache-control"] == "max-age=3600, immutable, public" - assert response.headers["content-length"] == "1207" + assert int(response.headers["content-length"]) > 100 assert response.headers["content-type"] == "image/png" From 150967d98ef6c5b6064587e7ed30cbdd9b992b8e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 20 Jan 2022 10:43:15 -0800 Subject: [PATCH 0592/1866] Hand-edited pixel favicon, refs #1603 --- datasette/static/favicon.png | Bin 1358 -> 208 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/datasette/static/favicon.png b/datasette/static/favicon.png index a27139ae6ecb09660f96d262d3779de29f7b5d4f..4993163f703c425c6512dec380dafe2ca52051a7 100644 GIT binary patch literal 208 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnF3?v&v(vJfvi2$DvS0Mc#47TsQvvJdnm8(v& zGBmL;l)e{q>Icd(l?3?(GYEgu?Rf^|rFyzJhE&{2KERTb>7mfr*l2iDf^!C=sKnw1 z79FNc4F-(`OP(xAS+ay@QOJ}j3QP)SnoLihzI2EP<5Qdv5^(o|!kVa_4lxb=X6adN yPl8f;beC`>@CQ#ili=ZWZBd)t&CVH}j0{Q7(vQX;{-qAIkipZ{&t;ucLK6Tps7T!a literal 1358 zcmV-U1+n^xP)<h;3K|Lk000e1NJLTq004jh004jp0{{R3^x%>C0000;P)t-s0001% zmb`Fqr$j`OPfwIOI*uG1hYt>g|NsB|{QdCp_1@p^(bD6=!P>XE+_AIRsjA42k-T+v zrB+v&Dk+Hr0*3$qfB*n>9JfE)00008bW%=J07QSN(*FMbUcFRk000E7Nkl<Zc-rk- zje4sf486525v_s>@%`WQlKxzNVr%CWzi#uW>voplSwP4@{_OZL9ExLB6vg4y?TeGK zxnWMl)7zgdqJ`^`=cF%a;XEnfZ~_DX0hkjY04$p0#Qg^g4mhGP`9S3ZfVHOsyFUU7 z#G%tFCv-O;3Iy9H0!|1B<DlF8=+1O68`nZ%>HhuALbX|-?j<*B7==YsgcaaKiLv)L zR~e4ruNx2mhbRC<c#EtdAW$ucAO!R?YO?W`YeNO!G#vuildl268^F5pk3*oYm4N2p zYrq0_83Vw}>_!pP!O;Q=uhM_!kwFpBR0sbAoGjp31uE6rmB2!vqppAi3=e@S_*n0v zGty(YfGe8&)i!jlbNTZ8k6gf6ZwrwM*c>(yU^fP)5dafI5T+Q;E%LO90Kjouuqqgu z7@dQT7){iyRiobWOrHWV_GJXwv`t$_RNXqRjB#<?CGl(>SH)J`Q=U;o0<f1M%8L4n zx(vCRdXy@nWaF$ZW$dZi#Iw9Ot0pO*3t)c-6%*nrK*nY_0TC+2JZ}O*W>$dY0_27Q z$RHIUH;sTSKQ96sw}B@SP)h{JpA><;4XoA%KC&m=ek@?4VT8>`!{HJ`ce-oqfT)#Y zG_zt{LtImb)Rd<|@j4?y#CNGBW;`PT5d#3;r52f4oE0EAMGWT_dBSW`iX%=j3pmE` zD&UQ<=^Ru<DWcLw=qNqm+0I&-$;VLAHS-vHBU#G(V<>&E_v$f}|CN-MwPV)mi1yKN z&k6T7@C`De4~@XjZUetOC*%{AUGI*Bany_z!BPFZm06OIaB4as!7nEye4bD&o+lK` z<_Xc_#$W<qTUbnU!YV?ol+u@ox@gB$F<Py3lXz<8**GheSWP~LlJ|r%?T9=@pN+Gk zoG|Ues(+lkbwoZVeBKe2RJgpIkDidoJNFRqps2h@N93)1M|3~@VC2i+bHe|~2Y!77 z+Fzd$?QI~d067%_`Fg!TJ{KUpBNrgQK^u5?rnKb(5>G!Ey%GRu`WS=5JJd8cZ<Q`y zTjVo)lhgt;ob<GBtR26(PQ1v|@(2XSU3SEICgGNNj>sZlW@~p?wG9Bmwok!=u6XmF za9=zA4x{0o59~SN-Uc%Fgx*9x%?T?{h5sFgo{D*{mJ_}|7*$_P$@mOOwek#!nw%k- z&xkn1IkU*qe%uR<%OW5Mi#%ceeL~O7P>ama&b-vxJEU{P4q7v=TjVJa)|f4CwtJt@ zPtyg|m(~l)4}YI<kkdJ#XVw*9*n&m1?PwUymLDwu)}_p!5BTwE6c&nyfC8);n^JBT z0a@zN1UoDa2B{Hjm=jh80nfoUCkui?kUs)G0MS;bf&@`3$lV5f3jj2Sc&C@32SLj0 z5pbV1B2WO*tyXBRWj$LtUF9W~t7=36I8XpQ3PB3Su9lfsPUWQ^5qt$i3P6n5cPe;C zzD?heS51hvBaoGc=AIql{>xz;1prQLx-0;}3cxWlgs^^}S-R&EEeN*$DIOO*C~-R< zX5L0jdwPaSLA1l+>8@9c&tY)|*l^D)M{zuxv&jcn`mvbX`kWVkc6=QE0tPO=y`~{k QD*ylh07*qoM6N<$g1B&Oe*gdg From ffca55dfd7cc9b53522c2e5a2fa1ff67c9beadf2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 20 Jan 2022 14:40:44 -0800 Subject: [PATCH 0593/1866] Show link to /stable/ on /latest/ pages, refs #1608 --- docs/_templates/layout.html | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index db16b428..785cdc7c 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -26,3 +26,36 @@ {% include "searchbox.html" %} {% endblock %} + +{% block footer %} +{{ super() }} +<script> +jQuery(function ($) { + // Show banner linking to /stable/ if this is a /latest/ page + if (!/\/latest\//.test(location.pathname)) { + return; + } + var stableUrl = location.pathname.replace("/latest/", "/stable/"); + // Check it's not a 404 + fetch(stableUrl, { method: "HEAD" }).then((response) => { + if (response.status == 200) { + var warning = $( + `<div class="admonition warning"> + <p class="first admonition-title">Note</p> + <p class="last"> + This documentation covers the <strong>development version</strong> of Datasette.</p> + <p>See <a href="${stableUrl}">this page</a> for the current stable release. + </p> + </div>` + ); + warning.find("a").attr("href", stableUrl); + var body = $("div.body"); + if (!body.length) { + body = $("div.document"); + } + body.prepend(warning); + } + }); +}); +</script> +{% endblock %} From d194db4204b732af57138e1fb0924ec77354dd58 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 20 Jan 2022 18:01:47 -0800 Subject: [PATCH 0594/1866] Output pip freeze to show installed packages, refs #1609 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78c289bb..2caf9447 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,7 @@ jobs: - name: Install dependencies run: | pip install -e '.[test]' + pip freeze - name: Run tests run: | pytest -n auto -m "not serial" From 68cc1e2dbb0b841af7a7691ea6b4e7d31b09cc5e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 25 Jan 2022 10:28:05 -0800 Subject: [PATCH 0595/1866] Move queries to top of database page, refs #1612 --- datasette/templates/database.html | 24 ++++++++++++++---------- tests/test_html.py | 30 +++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 2d182d1b..c1e39bd1 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -67,10 +67,23 @@ </div> {% endif %} +{% if queries %} + <h2 id="queries">Queries</h2> + <ul class="bullets"> + {% for query in queries %} + <li><a href="{{ urls.query(database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}</li> + {% endfor %} + </ul> +{% endif %} + +{% if tables %} +<h2 id="tables">Tables</h2> +{% endif %} + {% for table in tables %} {% if show_hidden or not table.hidden %} <div class="db-table"> - <h2><a href="{{ urls.table(database, table.name) }}">{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if table.hidden %}<em> (hidden)</em>{% endif %}</h2> + <h3><a href="{{ urls.table(database, table.name) }}">{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if table.hidden %}<em> (hidden)</em>{% endif %}</h3> <p><em>{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}</em></p> <p>{% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}</p> </div> @@ -90,15 +103,6 @@ </ul> {% endif %} -{% if queries %} - <h2 id="queries">Queries</h2> - <ul class="bullets"> - {% for query in queries %} - <li><a href="{{ urls.query(database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}</li> - {% endfor %} - </ul> -{% endif %} - {% if allow_download %} <p class="download-sqlite">Download SQLite DB: <a href="{{ urls.database(database) }}.db">{{ database }}.db</a> <em>{{ format_bytes(size) }}</em></p> {% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index aa718857..1bbf335c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -110,12 +110,32 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash): def test_database_page(app_client): response = app_client.get("/fixtures") - assert ( - b"<p><em>pk, foreign_key_with_label, foreign_key_with_blank_label, " - b"foreign_key_with_no_label, foreign_key_compound_pk1, " - b"foreign_key_compound_pk2</em></p>" - ) in response.body soup = Soup(response.body, "html.parser") + # Should have a <textarea> for executing SQL + assert "<textarea" in response.text + + # And a list of tables + for fragment in ( + '<h2 id="tables">Tables</h2>', + '<h3><a href="/fixtures/sortable">sortable</a></h3>', + "<p><em>pk, foreign_key_with_label, foreign_key_with_blank_label, ", + ): + assert fragment in response.text + + # And views + views_ul = soup.find("h2", text="Views").find_next_sibling("ul") + assert views_ul is not None + assert [ + ("/fixtures/paginated_view", "paginated_view"), + ("/fixtures/searchable_view", "searchable_view"), + ( + "/fixtures/searchable_view_configured_by_metadata", + "searchable_view_configured_by_metadata", + ), + ("/fixtures/simple_view", "simple_view"), + ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) + + # And a list of canned queries queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ From 84391763a8d5911c387c9965c86c8d45f39b31fb Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 25 Jan 2022 10:39:03 -0800 Subject: [PATCH 0596/1866] Clarify that magic parameters don't work for custom SQL --- docs/sql_queries.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index f9a36490..010e3205 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -275,6 +275,8 @@ Magic parameters Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string. +These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. + Available magic parameters are: ``_actor_*`` - e.g. ``_actor_id``, ``_actor_name`` From 2aa686c6554bf6b8230eb5b3019574df6cc99225 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 26 Jan 2022 10:21:05 -0800 Subject: [PATCH 0597/1866] It's not a weekly newsletter --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 107d81da..557d9290 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Live demo of current main: https://latest.datasette.io/ * Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) -Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. +Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. ## Installation From 3ef47a0896c7e63404a34e465b7160c80eaa571d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 27 Nov 2021 12:08:42 -0800 Subject: [PATCH 0598/1866] Link rel=alternate header for tables and rows Also added Access-Control-Expose-Headers: Link to --cors mode. Closes #1533 Refs https://github.com/simonw/datasette-notebook/issues/2 LL# metadata.json.1 --- datasette/templates/base.html | 2 +- datasette/templates/row.html | 3 ++- datasette/templates/table.html | 3 ++- datasette/utils/__init__.py | 1 + datasette/views/base.py | 12 ++++++++++-- datasette/views/table.py | 22 +++++++++++++++++++++- docs/json_api.rst | 20 ++++++++++++++++++++ tests/test_api.py | 1 + tests/test_table_html.py | 28 ++++++++++++++++++++++++++++ 9 files changed, 86 insertions(+), 6 deletions(-) diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c9aa7e31..836b7bb7 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -10,7 +10,7 @@ {% for url in extra_js_urls %} <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.sri %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script> {% endfor %} -{% block extra_head %}{% endblock %} +{%- block extra_head %}{% endblock -%} </head> <body class="{% block body_class %}{% endblock %}"> <div class="not-footer"> diff --git a/datasette/templates/row.html b/datasette/templates/row.html index c86e979d..1ac16268 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -3,7 +3,8 @@ {% block title %}{{ database }}: {{ table }}{% endblock %} {% block extra_head %} -{{ super() }} +{{- super() -}} +<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> <style> @media only screen and (max-width: 576px) { {% for column in columns %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index e3c6f38d..403e1d5b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -3,7 +3,8 @@ {% block title %}{{ database }}: {{ table }}: {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} -{{ super() }} +{{- super() -}} +<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> <script src="{{ urls.static('table.js') }}" defer></script> <style> @media only screen and (max-width: 576px) { diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bc3155a5..dc4e1c99 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1094,3 +1094,4 @@ async def derive_named_parameters(db, sql): def add_cors_headers(headers): headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Allow-Headers"] = "Authorization" + headers["Access-Control-Expose-Headers"] = "Link" diff --git a/datasette/views/base.py b/datasette/views/base.py index b1cacb3f..a414892a 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -137,10 +137,18 @@ class BaseView: ], }, } + # Hacky cheat to add extra headers + headers = {} + if "_extra_headers" in context: + headers.update(context["_extra_headers"]) return Response.html( await self.ds.render_template( - template, template_context, request=request, view_name=self.name - ) + template, + template_context, + request=request, + view_name=self.name, + ), + headers=headers, ) @classmethod diff --git a/datasette/views/table.py b/datasette/views/table.py index 77fb2850..6bbee352 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -17,6 +17,7 @@ from datasette.utils import ( is_url, path_from_row_pks, path_with_added_args, + path_with_format, path_with_removed_args, path_with_replaced_args, to_css_class, @@ -850,7 +851,12 @@ class TableView(RowTableShared): for table_column in table_columns if table_column not in columns ] + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path(path_with_format(request=request, format="json")), + ) d = { + "alternate_url_json": alternate_url_json, "table_actions": table_actions, "use_rowid": use_rowid, "filters": filters, @@ -881,6 +887,11 @@ class TableView(RowTableShared): "metadata": metadata, "view_definition": await db.get_view_definition(table), "table_definition": await db.get_table_definition(table), + "_extra_headers": { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, } d.update(extra_context_from_filters) return d @@ -964,8 +975,12 @@ class RowView(RowTableShared): ) for column in display_columns: column["sortable"] = False - + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path(path_with_format(request=request, format="json")), + ) return { + "alternate_url_json": alternate_url_json, "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), @@ -980,6 +995,11 @@ class RowView(RowTableShared): .get(database, {}) .get("tables", {}) .get(table, {}), + "_extra_headers": { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, } data = { diff --git a/docs/json_api.rst b/docs/json_api.rst index bd55c163..b5a6744b 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -14,6 +14,7 @@ served with the following additional HTTP headers:: Access-Control-Allow-Origin: * Access-Control-Allow-Headers: Authorization + Access-Control-Expose-Headers: Link This means JavaScript running on any domain will be able to make cross-origin requests to fetch the data. @@ -435,3 +436,22 @@ looks like:: The column in the foreign key table that is used for the label can be specified in ``metadata.json`` - see :ref:`label_columns`. + +.. _json_api_discover_alternate: + +Discovering the JSON for a page +------------------------------- + +The :ref:`table <TableView>` and :ref:`row <RowView>` HTML pages both provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism. + +You can find this near the top of those pages, looking like this: + +.. code-block:: python + + <link rel="alternate" + type="application/json+datasette" + href="https://latest.datasette.io/fixtures/sortable.json"> + +The JSON URL is also made available in a ``Link`` HTTP header for the page:: + + Link: https://latest.datasette.io/fixtures/sortable.json; rel="alternate"; type="application/json+datasette" diff --git a/tests/test_api.py b/tests/test_api.py index 47ec3a8c..9741ffc5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -977,6 +977,7 @@ def test_cors(app_client_with_cors, path, status_code): assert response.status == status_code assert response.headers["Access-Control-Allow-Origin"] == "*" assert response.headers["Access-Control-Allow-Headers"] == "Authorization" + assert response.headers["Access-Control-Expose-Headers"] == "Link" @pytest.mark.parametrize( diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 021268c3..7d08d230 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1069,3 +1069,31 @@ def test_table_page_title(app_client, path, expected): response = app_client.get(path) title = Soup(response.text, "html.parser").find("title").text assert title == expected + + +@pytest.mark.parametrize( + "path,expected", + ( + ( + "/fixtures/table%2Fwith%2Fslashes.csv", + "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json", + ), + ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), + ( + "/fixtures/no_primary_key/1", + "http://localhost/fixtures/no_primary_key/1.json", + ), + ), +) +def test_alternate_url_json(app_client, path, expected): + response = app_client.get(path) + link = response.headers["link"] + assert link == '{}; rel="alternate"; type="application/json+datasette"'.format( + expected + ) + assert ( + '<link rel="alternate" type="application/json+datasette" href="{}">'.format( + expected + ) + in response.text + ) From b72b2423c79dea4600b2337949db98269d0b6215 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 2 Feb 2022 13:21:11 -0800 Subject: [PATCH 0599/1866] rel=alternate JSON for queries and database pages, closes #1620 --- datasette/templates/database.html | 3 ++- datasette/templates/query.html | 3 ++- datasette/views/database.py | 32 ++++++++++++++++++++++ docs/json_api.rst | 2 +- tests/test_canned_queries.py | 9 +++++++ tests/test_html.py | 45 +++++++++++++++++++++++++++++++ tests/test_table_html.py | 28 ------------------- 7 files changed, 91 insertions(+), 31 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index c1e39bd1..8f0c65d7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -3,7 +3,8 @@ {% block title %}{{ database }}{% endblock %} {% block extra_head %} -{{ super() }} +{{- super() -}} +<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> {% include "_codemirror.html" %} {% endblock %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 75f7f1b1..d0121976 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -3,7 +3,8 @@ {% block title %}{{ database }}{% if query and query.sql %}: {{ query.sql }}{% endif %}{% endblock %} {% block extra_head %} -{{ super() }} +{{- super() -}} +<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> {% if columns %} <style> @media only screen and (max-width: 576px) { diff --git a/datasette/views/database.py b/datasette/views/database.py index e26706e7..f3641dc5 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -123,6 +123,10 @@ class DatabaseView(DataView): attached_databases = [d.name for d in await db.attached_databases()] + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path(path_with_format(request=request, format="json")), + ) return ( { "database": database, @@ -140,6 +144,7 @@ class DatabaseView(DataView): ), }, { + "alternate_url_json": alternate_url_json, "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, @@ -148,6 +153,11 @@ class DatabaseView(DataView): and not db.is_mutable and not db.is_memory, "attached_databases": attached_databases, + "_extra_headers": { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, }, (f"database-{to_css_class(database)}.html", "database.html"), ) @@ -308,7 +318,14 @@ class QueryView(DataView): else: async def extra_template(): + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path( + path_with_format(request=request, format="json") + ), + ) return { + "alternate_url_json": alternate_url_json, "request": request, "path_with_added_args": path_with_added_args, "path_with_removed_args": path_with_removed_args, @@ -316,6 +333,11 @@ class QueryView(DataView): "canned_query": canned_query, "success_message": request.args.get("_success") or "", "canned_write": True, + "_extra_headers": { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, } return ( @@ -448,7 +470,12 @@ class QueryView(DataView): show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) show_hide_text = "hide" hide_sql = show_hide_text == "show" + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path(path_with_format(request=request, format="json")), + ) return { + "alternate_url_json": alternate_url_json, "display_rows": display_rows, "custom_sql": True, "named_parameter_values": named_parameter_values, @@ -462,6 +489,11 @@ class QueryView(DataView): "show_hide_text": show_hide_text, "show_hide_hidden": markupsafe.Markup(show_hide_hidden), "hide_sql": hide_sql, + "_extra_headers": { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, } return ( diff --git a/docs/json_api.rst b/docs/json_api.rst index b5a6744b..4f9eaddb 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -442,7 +442,7 @@ in ``metadata.json`` - see :ref:`label_columns`. Discovering the JSON for a page ------------------------------- -The :ref:`table <TableView>` and :ref:`row <RowView>` HTML pages both provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism. +The :ref:`database <DatabaseView>`, :ref:`table <TableView>`, :ref:`custom/canned query <sql>` and :ref:`row <RowView>` HTML pages all provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism. You can find this near the top of those pages, looking like this: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index c5ccaf5c..5abffdcc 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -364,3 +364,12 @@ def test_canned_write_custom_template(canned_write_client): in response.text ) assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text + # And test for link rel=alternate while we're here: + assert ( + '<link rel="alternate" type="application/json+datasette" href="http://localhost/data/update_name.json">' + in response.text + ) + assert ( + response.headers["link"] + == 'http://localhost/data/update_name.json; rel="alternate"; type="application/json+datasette"' + ) diff --git a/tests/test_html.py b/tests/test_html.py index 1bbf335c..273e4914 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -870,3 +870,48 @@ def test_trace_correctly_escaped(app_client): response = app_client.get("/fixtures?sql=select+'<h1>Hello'&_trace=1") assert "select '<h1>Hello" not in response.text assert "select '<h1>Hello" in response.text + + +@pytest.mark.parametrize( + "path,expected", + ( + # Table page + ( + "/fixtures/table%2Fwith%2Fslashes.csv", + "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json", + ), + ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), + # Row page + ( + "/fixtures/no_primary_key/1", + "http://localhost/fixtures/no_primary_key/1.json", + ), + # Database index page + ( + "/fixtures", + "http://localhost/fixtures.json", + ), + # Custom query page + ( + "/fixtures?sql=select+*+from+facetable", + "http://localhost/fixtures.json?sql=select+*+from+facetable", + ), + # Canned query page + ( + "/fixtures/neighborhood_search?text=town", + "http://localhost/fixtures/neighborhood_search.json?text=town", + ), + ), +) +def test_alternate_url_json(app_client, path, expected): + response = app_client.get(path) + link = response.headers["link"] + assert link == '{}; rel="alternate"; type="application/json+datasette"'.format( + expected + ) + assert ( + '<link rel="alternate" type="application/json+datasette" href="{}">'.format( + expected + ) + in response.text + ) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 7d08d230..021268c3 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1069,31 +1069,3 @@ def test_table_page_title(app_client, path, expected): response = app_client.get(path) title = Soup(response.text, "html.parser").find("title").text assert title == expected - - -@pytest.mark.parametrize( - "path,expected", - ( - ( - "/fixtures/table%2Fwith%2Fslashes.csv", - "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json", - ), - ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), - ( - "/fixtures/no_primary_key/1", - "http://localhost/fixtures/no_primary_key/1.json", - ), - ), -) -def test_alternate_url_json(app_client, path, expected): - response = app_client.get(path) - link = response.headers["link"] - assert link == '{}; rel="alternate"; type="application/json+datasette"'.format( - expected - ) - assert ( - '<link rel="alternate" type="application/json+datasette" href="{}">'.format( - expected - ) - in response.text - ) From 8d5779acf0041cfd0db7f68f468419f9008b86ec Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 2 Feb 2022 13:32:47 -0800 Subject: [PATCH 0600/1866] Refactored alternate_url_json mechanism, refs #1620, #1533 --- datasette/templates/base.html | 3 +++ datasette/templates/database.html | 1 - datasette/templates/query.html | 1 - datasette/templates/row.html | 1 - datasette/templates/table.html | 1 - datasette/views/base.py | 17 +++++++++++++--- datasette/views/database.py | 32 ------------------------------- datasette/views/table.py | 20 ------------------- docs/json_api.rst | 4 ++-- tests/test_html.py | 9 ++++++++- 10 files changed, 27 insertions(+), 62 deletions(-) diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 836b7bb7..c3a71acb 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -10,6 +10,9 @@ {% for url in extra_js_urls %} <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.sri %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script> {% endfor %} +{%- if alternate_url_json -%} + <link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> +{%- endif -%} {%- block extra_head %}{% endblock -%} </head> <body class="{% block body_class %}{% endblock %}"> diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 8f0c65d7..e76bc49e 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -4,7 +4,6 @@ {% block extra_head %} {{- super() -}} -<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> {% include "_codemirror.html" %} {% endblock %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index d0121976..8c920527 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -4,7 +4,6 @@ {% block extra_head %} {{- super() -}} -<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> {% if columns %} <style> @media only screen and (max-width: 576px) { diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 1ac16268..10770ce9 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -4,7 +4,6 @@ {% block extra_head %} {{- super() -}} -<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> <style> @media only screen and (max-width: 576px) { {% for column in columns %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 403e1d5b..81bd044a 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -4,7 +4,6 @@ {% block extra_head %} {{- super() -}} -<link rel="alternate" type="application/json+datasette" href="{{ alternate_url_json }}"> <script src="{{ urls.static('table.js') }}" defer></script> <style> @media only screen and (max-width: 576px) { diff --git a/datasette/views/base.py b/datasette/views/base.py index a414892a..c74d6141 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -55,6 +55,7 @@ class DatasetteError(Exception): class BaseView: ds = None + has_json_alternate = True def __init__(self, datasette): self.ds = datasette @@ -137,10 +138,20 @@ class BaseView: ], }, } - # Hacky cheat to add extra headers headers = {} - if "_extra_headers" in context: - headers.update(context["_extra_headers"]) + if self.has_json_alternate: + alternate_url_json = self.ds.absolute_url( + request, + self.ds.urls.path(path_with_format(request=request, format="json")), + ) + template_context["alternate_url_json"] = alternate_url_json + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) return Response.html( await self.ds.render_template( template, diff --git a/datasette/views/database.py b/datasette/views/database.py index f3641dc5..e26706e7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -123,10 +123,6 @@ class DatabaseView(DataView): attached_databases = [d.name for d in await db.attached_databases()] - alternate_url_json = self.ds.absolute_url( - request, - self.ds.urls.path(path_with_format(request=request, format="json")), - ) return ( { "database": database, @@ -144,7 +140,6 @@ class DatabaseView(DataView): ), }, { - "alternate_url_json": alternate_url_json, "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, @@ -153,11 +148,6 @@ class DatabaseView(DataView): and not db.is_mutable and not db.is_memory, "attached_databases": attached_databases, - "_extra_headers": { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - }, }, (f"database-{to_css_class(database)}.html", "database.html"), ) @@ -318,14 +308,7 @@ class QueryView(DataView): else: async def extra_template(): - alternate_url_json = self.ds.absolute_url( - request, - self.ds.urls.path( - path_with_format(request=request, format="json") - ), - ) return { - "alternate_url_json": alternate_url_json, "request": request, "path_with_added_args": path_with_added_args, "path_with_removed_args": path_with_removed_args, @@ -333,11 +316,6 @@ class QueryView(DataView): "canned_query": canned_query, "success_message": request.args.get("_success") or "", "canned_write": True, - "_extra_headers": { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - }, } return ( @@ -470,12 +448,7 @@ class QueryView(DataView): show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) show_hide_text = "hide" hide_sql = show_hide_text == "show" - alternate_url_json = self.ds.absolute_url( - request, - self.ds.urls.path(path_with_format(request=request, format="json")), - ) return { - "alternate_url_json": alternate_url_json, "display_rows": display_rows, "custom_sql": True, "named_parameter_values": named_parameter_values, @@ -489,11 +462,6 @@ class QueryView(DataView): "show_hide_text": show_hide_text, "show_hide_hidden": markupsafe.Markup(show_hide_hidden), "hide_sql": hide_sql, - "_extra_headers": { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - }, } return ( diff --git a/datasette/views/table.py b/datasette/views/table.py index 6bbee352..be9e9c3b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -851,12 +851,7 @@ class TableView(RowTableShared): for table_column in table_columns if table_column not in columns ] - alternate_url_json = self.ds.absolute_url( - request, - self.ds.urls.path(path_with_format(request=request, format="json")), - ) d = { - "alternate_url_json": alternate_url_json, "table_actions": table_actions, "use_rowid": use_rowid, "filters": filters, @@ -887,11 +882,6 @@ class TableView(RowTableShared): "metadata": metadata, "view_definition": await db.get_view_definition(table), "table_definition": await db.get_table_definition(table), - "_extra_headers": { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - }, } d.update(extra_context_from_filters) return d @@ -975,12 +965,7 @@ class RowView(RowTableShared): ) for column in display_columns: column["sortable"] = False - alternate_url_json = self.ds.absolute_url( - request, - self.ds.urls.path(path_with_format(request=request, format="json")), - ) return { - "alternate_url_json": alternate_url_json, "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), @@ -995,11 +980,6 @@ class RowView(RowTableShared): .get(database, {}) .get("tables", {}) .get(table, {}), - "_extra_headers": { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - }, } data = { diff --git a/docs/json_api.rst b/docs/json_api.rst index 4f9eaddb..aa6fcdaa 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -442,9 +442,9 @@ in ``metadata.json`` - see :ref:`label_columns`. Discovering the JSON for a page ------------------------------- -The :ref:`database <DatabaseView>`, :ref:`table <TableView>`, :ref:`custom/canned query <sql>` and :ref:`row <RowView>` HTML pages all provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism. +Most of the HTML pages served by Datasette provide a mechanism for discovering their JSON equivalents using the HTML ``link`` mechanism. -You can find this near the top of those pages, looking like this: +You can find this near the top of the source code of those pages, looking like this: .. code-block:: python diff --git a/tests/test_html.py b/tests/test_html.py index 273e4914..4b6cbd13 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -875,12 +875,14 @@ def test_trace_correctly_escaped(app_client): @pytest.mark.parametrize( "path,expected", ( + # Instance index page + ("/", "http://localhost/.json"), # Table page + ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), ( "/fixtures/table%2Fwith%2Fslashes.csv", "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json", ), - ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), # Row page ( "/fixtures/no_primary_key/1", @@ -901,6 +903,11 @@ def test_trace_correctly_escaped(app_client): "/fixtures/neighborhood_search?text=town", "http://localhost/fixtures/neighborhood_search.json?text=town", ), + # /-/ pages + ( + "/-/plugins", + "http://localhost/-/plugins.json", + ), ), ) def test_alternate_url_json(app_client, path, expected): From 23a09b0f6af33c52acf8c1d9002fe475b42fee10 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 2 Feb 2022 13:48:52 -0800 Subject: [PATCH 0601/1866] Remove JSON rel=alternate from some pages, closes #1623 --- datasette/views/special.py | 6 ++++++ tests/test_html.py | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/datasette/views/special.py b/datasette/views/special.py index 3cb626a5..cdd530f0 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -43,6 +43,7 @@ class JsonDataView(BaseView): class PatternPortfolioView(BaseView): name = "patterns" + has_json_alternate = False async def get(self, request): await self.check_permission(request, "view-instance") @@ -51,6 +52,7 @@ class PatternPortfolioView(BaseView): class AuthTokenView(BaseView): name = "auth_token" + has_json_alternate = False async def get(self, request): token = request.args.get("token") or "" @@ -69,6 +71,7 @@ class AuthTokenView(BaseView): class LogoutView(BaseView): name = "logout" + has_json_alternate = False async def get(self, request): if not request.actor: @@ -88,6 +91,7 @@ class LogoutView(BaseView): class PermissionsDebugView(BaseView): name = "permissions_debug" + has_json_alternate = False async def get(self, request): await self.check_permission(request, "view-instance") @@ -103,6 +107,7 @@ class PermissionsDebugView(BaseView): class AllowDebugView(BaseView): name = "allow_debug" + has_json_alternate = False async def get(self, request): errors = [] @@ -137,6 +142,7 @@ class AllowDebugView(BaseView): class MessagesDebugView(BaseView): name = "messages_debug" + has_json_alternate = False async def get(self, request): await self.check_permission(request, "view-instance") diff --git a/tests/test_html.py b/tests/test_html.py index 4b6cbd13..d5f4250d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -922,3 +922,15 @@ def test_alternate_url_json(app_client, path, expected): ) in response.text ) + + +@pytest.mark.parametrize( + "path", + ("/-/patterns", "/-/messages", "/-/allow-debug", "/fixtures.db"), +) +def test_no_alternate_url_json(app_client, path): + response = app_client.get(path) + assert "link" not in response.headers + assert ( + '<link rel="alternate" type="application/json+datasette"' not in response.text + ) From a9d8824617268c4d214dd3be2174ac452044f737 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 2 Feb 2022 13:58:52 -0800 Subject: [PATCH 0602/1866] Test against Python 3.11-dev Closes #1621 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2caf9447..478e1f34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From b5e6b1a9e1332fca3effe45d55dd06ee4249f163 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Feb 2022 14:23:51 -0800 Subject: [PATCH 0603/1866] Bump black from 21.12b0 to 22.1.0 (#1616) Bumps [black](https://github.com/psf/black) from 21.12b0 to 22.1.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits/22.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dade0a88..6accaa30 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.10,<0.17", "beautifulsoup4>=4.8.1,<4.11.0", - "black==21.12b0", + "black==22.1.0", "pytest-timeout>=1.4.2,<2.1", "trustme>=0.7,<0.10", "cogapp>=3.3.0", From 1af1041f91a9b91b321078d354132d1df5204660 Mon Sep 17 00:00:00 2001 From: Robert Christie <robc@pobox.com> Date: Thu, 3 Feb 2022 01:58:35 +0000 Subject: [PATCH 0604/1866] Jinja template_name should use "/" even on Windows (#1617) Closes #1545. Thanks, Robert Christie --- datasette/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 09d7d034..7bdf076c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1212,9 +1212,10 @@ class DatasetteRouter: else: # Is there a pages/* template matching this path? route_path = request.scope.get("route_path", request.scope["path"]) - template_path = os.path.join("pages", *route_path.split("/")) + ".html" + # Jinja requires template names to use "/" even on Windows + template_name = "pages" + route_path + ".html" try: - template = self.ds.jinja_env.select_template([template_path]) + template = self.ds.jinja_env.select_template([template_name]) except TemplateNotFound: template = None if template is None: From ac239d34ab2de6987afac43f5d38b576b26e9457 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 4 Feb 2022 20:45:13 -0800 Subject: [PATCH 0605/1866] Refactor test_trace into separate test module, refs #1576 --- tests/test_api.py | 51 ------------------------------------------ tests/test_tracer.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 51 deletions(-) create mode 100644 tests/test_tracer.py diff --git a/tests/test_api.py b/tests/test_api.py index 9741ffc5..57471af2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -911,57 +911,6 @@ def test_config_force_https_urls(): assert client.ds._last_request.scheme == "https" -@pytest.mark.parametrize("trace_debug", (True, False)) -def test_trace(trace_debug): - with make_app_client(settings={"trace_debug": trace_debug}) as client: - response = client.get("/fixtures/simple_primary_key.json?_trace=1") - assert response.status == 200 - - data = response.json - if not trace_debug: - assert "_trace" not in data - return - - assert "_trace" in data - trace_info = data["_trace"] - assert isinstance(trace_info["request_duration_ms"], float) - assert isinstance(trace_info["sum_trace_duration_ms"], float) - assert isinstance(trace_info["num_traces"], int) - assert isinstance(trace_info["traces"], list) - traces = trace_info["traces"] - assert len(traces) == trace_info["num_traces"] - for trace in traces: - assert isinstance(trace["type"], str) - assert isinstance(trace["start"], float) - assert isinstance(trace["end"], float) - assert trace["duration_ms"] == (trace["end"] - trace["start"]) * 1000 - assert isinstance(trace["traceback"], list) - assert isinstance(trace["database"], str) - assert isinstance(trace["sql"], str) - assert isinstance(trace.get("params"), (list, dict, None.__class__)) - - sqls = [trace["sql"] for trace in traces if "sql" in trace] - # There should be a mix of different types of SQL statement - expected = ( - "CREATE TABLE ", - "PRAGMA ", - "INSERT OR REPLACE INTO ", - "INSERT INTO", - "select ", - ) - for prefix in expected: - assert any( - sql.startswith(prefix) for sql in sqls - ), "No trace beginning with: {}".format(prefix) - - # Should be at least one executescript - assert any(trace for trace in traces if trace.get("executescript")) - # And at least one executemany - execute_manys = [trace for trace in traces if trace.get("executemany")] - assert execute_manys - assert all(isinstance(trace["count"], int) for trace in execute_manys) - - @pytest.mark.parametrize( "path,status_code", [ diff --git a/tests/test_tracer.py b/tests/test_tracer.py new file mode 100644 index 00000000..20a4427e --- /dev/null +++ b/tests/test_tracer.py @@ -0,0 +1,53 @@ +import pytest +from .fixtures import make_app_client + + +@pytest.mark.parametrize("trace_debug", (True, False)) +def test_trace(trace_debug): + with make_app_client(settings={"trace_debug": trace_debug}) as client: + response = client.get("/fixtures/simple_primary_key.json?_trace=1") + assert response.status == 200 + + data = response.json + if not trace_debug: + assert "_trace" not in data + return + + assert "_trace" in data + trace_info = data["_trace"] + assert isinstance(trace_info["request_duration_ms"], float) + assert isinstance(trace_info["sum_trace_duration_ms"], float) + assert isinstance(trace_info["num_traces"], int) + assert isinstance(trace_info["traces"], list) + traces = trace_info["traces"] + assert len(traces) == trace_info["num_traces"] + for trace in traces: + assert isinstance(trace["type"], str) + assert isinstance(trace["start"], float) + assert isinstance(trace["end"], float) + assert trace["duration_ms"] == (trace["end"] - trace["start"]) * 1000 + assert isinstance(trace["traceback"], list) + assert isinstance(trace["database"], str) + assert isinstance(trace["sql"], str) + assert isinstance(trace.get("params"), (list, dict, None.__class__)) + + sqls = [trace["sql"] for trace in traces if "sql" in trace] + # There should be a mix of different types of SQL statement + expected = ( + "CREATE TABLE ", + "PRAGMA ", + "INSERT OR REPLACE INTO ", + "INSERT INTO", + "select ", + ) + for prefix in expected: + assert any( + sql.startswith(prefix) for sql in sqls + ), "No trace beginning with: {}".format(prefix) + + # Should be at least one executescript + assert any(trace for trace in traces if trace.get("executescript")) + # And at least one executemany + execute_manys = [trace for trace in traces if trace.get("executemany")] + assert execute_manys + assert all(isinstance(trace["count"], int) for trace in execute_manys) From da53e0360da4771ffb56a8e3eb3f7476f3168299 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 4 Feb 2022 21:19:49 -0800 Subject: [PATCH 0606/1866] tracer.trace_child_tasks() for asyncio.gather tracing Also added documentation for datasette.tracer module. Closes #1576 --- datasette/tracer.py | 20 +++++++---- docs/internals.rst | 71 ++++++++++++++++++++++++++++++++++++++ tests/plugins/my_plugin.py | 12 +++++++ tests/test_tracer.py | 15 ++++++++ 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 6703f060..fc7338b0 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,5 +1,6 @@ import asyncio from contextlib import contextmanager +from contextvars import ContextVar from markupsafe import escape import time import json @@ -9,20 +10,25 @@ tracers = {} TRACE_RESERVED_KEYS = {"type", "start", "end", "duration_ms", "traceback"} - -# asyncio.current_task was introduced in Python 3.7: -for obj in (asyncio, asyncio.Task): - current_task = getattr(obj, "current_task", None) - if current_task is not None: - break +trace_task_id = ContextVar("trace_task_id", default=None) def get_task_id(): + current = trace_task_id.get(None) + if current is not None: + return current try: loop = asyncio.get_event_loop() except RuntimeError: return None - return id(current_task(loop=loop)) + return id(asyncio.current_task(loop=loop)) + + +@contextmanager +def trace_child_tasks(): + token = trace_task_id.set(get_task_id()) + yield + trace_task_id.reset(token) @contextmanager diff --git a/docs/internals.rst b/docs/internals.rst index 6a5666fd..a5dbdfb4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -864,3 +864,74 @@ parse_metadata(content) This function accepts a string containing either JSON or YAML, expected to be of the format described in :ref:`metadata`. It returns a nested Python dictionary representing the parsed data from that string. If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception. + +.. _internals_tracer + +datasette.tracer +================ + +Running Datasette with ``--setting trace_debug 1`` enables trace debug output, which can then be viewed by adding ``?_trace=1`` to the query string for any page. + +You can see an example of this at the bottom of `latest.datasette.io/fixtures/facetable?_trace=1 <https://latest.datasette.io/fixtures/facetable?_trace=1>`__. The JSON output shows full details of every SQL query that was executed to generate the page. + +The `datasette-pretty-traces <https://datasette.io/plugins/datasette-pretty-traces>`__ plugin can be installed to provide a more readable display of this information. You can see `a demo of that here <https://latest-with-plugins.datasette.io/github/commits?_trace=1>`__. + +You can add your own custom traces to the JSON output using the ``trace()`` context manager. This takes a string that identifies the type of trace being recorded, and records any keyword arguments as additional JSON keys on the resulting trace object. + +The start and end time, duration and a traceback of where the trace was executed will be automatically attached to the JSON object. + +This example uses trace to record the start, end and duration of any HTTP GET requests made using the function: + +.. code-block:: python + + from datasette.tracer import trace + import httpx + + async def fetch_url(url): + with trace("fetch-url", url=url): + async with httpx.AsyncClient() as client: + return await client.get(url) + +.. _internals_tracer_trace_child_tasks + +Tracing child tasks +------------------- + +If your code uses a mechanism such as ``asyncio.gather()`` to execute code in additional tasks you may find that some of the traces are missing from the display. + +You can use the ``trace_child_tasks()`` context manager to ensure these child tasks are correctly handled. + +.. code-block:: python + + from datasette import tracer + + with tracer.trace_child_tasks(): + results = await asyncio.gather( + # ... async tasks here + ) + +This example uses the :ref:`register_routes() <plugin_register_routes>` plugin hook to add a page at ``/parallel-queries`` which executes two SQL queries in parallel using ``asyncio.gather()`` and returns their results. + +.. code-block:: python + + from datasette import hookimpl + from datasette import tracer + + @hookimpl + def register_routes(): + + async def parallel_queries(datasette): + db = datasette.get_database() + with tracer.trace_child_tasks(): + one, two = await asyncio.gather( + db.execute("select 1"), + db.execute("select 2"), + ) + return Response.json({"one": one.single_value(), "two": two.single_value()}) + + return [ + (r"/parallel-queries$", parallel_queries), + ] + + +Adding ``?_trace=1`` will show that the trace covers both of those child tasks. diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 75c76ea8..610cea17 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -1,5 +1,7 @@ +import asyncio from datasette import hookimpl from datasette.facets import Facet +from datasette import tracer from datasette.utils import path_with_added_args from datasette.utils.asgi import asgi_send_json, Response import base64 @@ -270,6 +272,15 @@ def register_routes(): def asgi_scope(scope): return Response.json(scope, default=repr) + async def parallel_queries(datasette): + db = datasette.get_database() + with tracer.trace_child_tasks(): + one, two = await asyncio.gather( + db.execute("select coalesce(sleep(0.1), 1)"), + db.execute("select coalesce(sleep(0.1), 2)"), + ) + return Response.json({"one": one.single_value(), "two": two.single_value()}) + return [ (r"/one/$", one), (r"/two/(?P<name>.*)$", two), @@ -281,6 +292,7 @@ def register_routes(): (r"/add-message/$", add_message), (r"/render-message/$", render_message), (r"/asgi-scope$", asgi_scope), + (r"/parallel-queries$", parallel_queries), ] diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 20a4427e..ceadee50 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -51,3 +51,18 @@ def test_trace(trace_debug): execute_manys = [trace for trace in traces if trace.get("executemany")] assert execute_manys assert all(isinstance(trace["count"], int) for trace in execute_manys) + + +def test_trace_parallel_queries(): + with make_app_client(settings={"trace_debug": True}) as client: + response = client.get("/parallel-queries?_trace=1") + assert response.status == 200 + + data = response.json + assert data["one"] == 1 + assert data["two"] == 2 + trace_info = data["_trace"] + traces = [trace for trace in trace_info["traces"] if "sql" in trace] + one, two = traces + # "two" should have started before "one" ended + assert two["start"] < one["end"] From 1c6b297e3ec288cf1f838796df499a9c21c31664 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 4 Feb 2022 21:28:35 -0800 Subject: [PATCH 0607/1866] Link to datasette.tracer from trace_debug docs, refs #1576 --- docs/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.rst b/docs/settings.rst index 7cc4bae0..da06d6a0 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -302,6 +302,8 @@ Some examples: * https://latest.datasette.io/?_trace=1 * https://latest.datasette.io/fixtures/roadside_attractions?_trace=1 +See :ref:`internals_tracer` for details on how to hook into this mechanism as a plugin author. + .. _setting_base_url: base_url From d25b55ab5e4d7368d374ea752b2232755869d40d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 5 Feb 2022 22:32:23 -0800 Subject: [PATCH 0608/1866] Fixed rST warnings --- docs/internals.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index a5dbdfb4..0b010295 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -865,7 +865,7 @@ This function accepts a string containing either JSON or YAML, expected to be of If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception. -.. _internals_tracer +.. _internals_tracer: datasette.tracer ================ @@ -892,7 +892,7 @@ This example uses trace to record the start, end and duration of any HTTP GET re async with httpx.AsyncClient() as client: return await client.get(url) -.. _internals_tracer_trace_child_tasks +.. _internals_tracer_trace_child_tasks: Tracing child tasks ------------------- From 8a25ea9bcae7ae4c9a4bd99f90c955828ff5676d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 5 Feb 2022 22:34:33 -0800 Subject: [PATCH 0609/1866] Implemented import shortcuts, closes #957 --- datasette/__init__.py | 2 ++ docs/internals.rst | 15 +++++++++++++++ docs/plugin_hooks.rst | 4 +++- tests/plugins/my_plugin.py | 9 +++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/datasette/__init__.py b/datasette/__init__.py index 0e59760a..faa36051 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,3 +1,5 @@ from datasette.version import __version_info__, __version__ # noqa +from datasette.utils.asgi import Forbidden, NotFound, Response # noqa +from datasette.utils import actor_matches_allow # noqa from .hookspecs import hookimpl # noqa from .hookspecs import hookspec # noqa diff --git a/docs/internals.rst b/docs/internals.rst index 0b010295..632f7d7a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -935,3 +935,18 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h Adding ``?_trace=1`` will show that the trace covers both of those child tasks. + +.. _internals_shortcuts: + +Import shortcuts +================ + +The following commonly used symbols can be imported directly from the ``datasette`` module: + +.. code-block:: python + + from datasette import Response + from datasette import Forbidden + from datasette import NotFound + from datasette import hookimpl + from datasette import actor_matches_allow diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 88e1def0..1308b704 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -542,7 +542,7 @@ Return a list of ``(regex, view_function)`` pairs, something like this: .. code-block:: python - from datasette.utils.asgi import Response + from datasette import Response import html @@ -582,6 +582,8 @@ The view function can be a regular function or an ``async def`` function, depend The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only). +It can also rase the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden. + See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin. Examples: `datasette-auth-github <https://datasette.io/plugins/datasette-auth-github>`__, `datasette-psutil <https://datasette.io/plugins/datasette-psutil>`__ diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 610cea17..1c9b0575 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -300,6 +300,15 @@ def register_routes(): def startup(datasette): datasette._startup_hook_fired = True + # And test some import shortcuts too + from datasette import Response + from datasette import Forbidden + from datasette import NotFound + from datasette import hookimpl + from datasette import actor_matches_allow + + _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) + @hookimpl def canned_queries(datasette, database, actor): From 9b83ff2ee4d3cb5bfc5cb09a3ec99819ac214434 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 5 Feb 2022 22:46:33 -0800 Subject: [PATCH 0610/1866] Fixed spelling of "raise" --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1308b704..a63d441e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -582,7 +582,7 @@ The view function can be a regular function or an ``async def`` function, depend The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only). -It can also rase the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden. +It can also raise the ``datasette.NotFound`` exception to return a 404 not found error, or the ``datasette.Forbidden`` exception for a 403 forbidden. See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin. From d9b508ffaa91f9f1840b366f5d282712d445f16b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 6 Feb 2022 22:30:00 -0800 Subject: [PATCH 0611/1866] @documented decorator plus unit test plus sphinx.ext.autodoc New mechanism for marking datasette.utils functions that should be covered by the documentation, then testing that they have indeed been documented. Also enabled sphinx.ext.autodoc which can now be used to embed the documented versions of those functions. Refs #1176 --- datasette/utils/__init__.py | 16 ++++++++++++++-- docs/conf.py | 2 +- docs/internals.rst | 11 +++++++++++ tests/test_docs.py | 18 +++++++++++++++++- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index dc4e1c99..610e916f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -12,6 +12,7 @@ import os import re import shlex import tempfile +import typing import time import types import shutil @@ -59,8 +60,17 @@ Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") ) +functions_marked_as_documented = [] -async def await_me_maybe(value): + +def documented(fn): + functions_marked_as_documented.append(fn) + return fn + + +@documented +async def await_me_maybe(value: typing.Any) -> typing.Any: + "If value is callable, call it. If awaitable, await it. Otherwise return it." if callable(value): value = value() if asyncio.iscoroutine(value): @@ -915,7 +925,9 @@ class BadMetadataError(Exception): pass -def parse_metadata(content): +@documented +def parse_metadata(content: str) -> dict: + "Detects if content is JSON or YAML and parses it appropriately." # content can be JSON or YAML try: return json.loads(content) diff --git a/docs/conf.py b/docs/conf.py index 89009ea9..d114bc52 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#"), diff --git a/docs/internals.rst b/docs/internals.rst index 632f7d7a..12ef5c54 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -865,6 +865,17 @@ This function accepts a string containing either JSON or YAML, expected to be of If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception. +.. autofunction:: datasette.utils.parse_metadata + +.. _internals_utils_await_me_maybe: + +await_me_maybe(value) +--------------------- + +Utility function for calling ``await`` on a return value if it is awaitable, otherwise returning the value. This is used by Datasette to support plugin hooks that can optionally return awaitable functions. Read more about this function in `The “await me maybe” pattern for Python asyncio <https://simonwillison.net/2020/Sep/2/await-me-maybe/>`__. + +.. autofunction:: datasette.utils.await_me_maybe + .. _internals_tracer: datasette.tracer diff --git a/tests/test_docs.py b/tests/test_docs.py index 0d17b8e3..cd5a6c13 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -2,7 +2,7 @@ Tests to ensure certain things are documented. """ from click.testing import CliRunner -from datasette import app +from datasette import app, utils from datasette.cli import cli from datasette.filters import Filters from pathlib import Path @@ -86,3 +86,19 @@ def documented_table_filters(): @pytest.mark.parametrize("filter", [f.key for f in Filters._filters]) def test_table_filters_are_documented(documented_table_filters, filter): assert filter in documented_table_filters + + +@pytest.fixture(scope="session") +def documented_fns(): + internals_rst = (docs_path / "internals.rst").read_text() + # Any line that starts .. _internals_utils_X + lines = internals_rst.split("\n") + prefix = ".. _internals_utils_" + return { + line.split(prefix)[1].split(":")[0] for line in lines if line.startswith(prefix) + } + + +@pytest.mark.parametrize("fn", utils.functions_marked_as_documented) +def test_functions_marked_with_documented_are_documented(documented_fns, fn): + assert fn.__name__ in documented_fns From fdce6f29e19c3c6b477b72f86e187abee9627b92 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 6 Feb 2022 22:38:27 -0800 Subject: [PATCH 0612/1866] Reconfigure ReadTheDocs, refs #1176 --- .readthedocs.yaml | 8 ++++++-- docs/readthedocs-requirements.txt | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) delete mode 100644 docs/readthedocs-requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 70db5313..60b73b30 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,5 +9,9 @@ sphinx: configuration: docs/conf.py python: - install: - - requirements: docs/readthedocs-requirements.txt + version: "3.9" + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt deleted file mode 100644 index db1851ad..00000000 --- a/docs/readthedocs-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -docutils<0.19 From 03305ea183b1534bc4cef3a721fe5f3700273b84 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 6 Feb 2022 22:40:47 -0800 Subject: [PATCH 0613/1866] Remove python.version, refs #1176 --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 60b73b30..e157fb9c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,6 @@ sphinx: configuration: docs/conf.py python: - version: "3.9" install: - method: pip path: . From 0cd982fc6af45b60e0c9306516dd412ae948c89b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Feb 2022 15:28:46 -0800 Subject: [PATCH 0614/1866] De-duplicate 'datasette db.db db.db', closes #1632 Refs https://github.com/simonw/datasette-publish-fly/pull/12 --- datasette/cli.py | 3 +++ tests/test_cli.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index 9d1b5ee5..61e7ce91 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -549,6 +549,9 @@ def serve( ) ) + # De-duplicate files so 'datasette db.db db.db' only attaches one /db + files = list(dict.fromkeys(files)) + try: ds = Datasette(files, **kwargs) except SpatialiteNotFound: diff --git a/tests/test_cli.py b/tests/test_cli.py index bbc5df30..3fbfdee2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -257,6 +257,7 @@ def test_serve_create(ensure_eventloop, tmpdir): def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): + "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() db_1_path = str(tmpdir / "db.db") nested = tmpdir / "nested" @@ -270,6 +271,17 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert {db["name"] for db in databases} == {"db", "db_2"} +def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): + "'datasette db.db db.db' should only attach one database, /db" + runner = CliRunner() + db_path = str(tmpdir / "db.db") + sqlite3.connect(db_path).execute("vacuum") + result = runner.invoke(cli, [db_path, db_path, "--get", "/-/databases.json"]) + assert result.exit_code == 0, result.output + databases = json.loads(result.output) + assert {db["name"] for db in databases} == {"db"} + + @pytest.mark.parametrize( "filename", ["test-database (1).sqlite", "database (1).sqlite"] ) From fa5fc327adbbf70656ac533912f3fc0526a3873d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Feb 2022 15:32:54 -0800 Subject: [PATCH 0615/1866] Release 0.60.2 Refs #1632 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index a4e340b3..91224615 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.60" +__version__ = "0.60.2" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7e2af39..c58c8444 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_60.2: + +0.60.2 (2022-02-07) +------------------- + +- Fixed a bug where Datasette would open the same file twice with two different database names if you ran ``datasette file.db file.db``. (:issue:`1632`) + +.. _v0_60.1: + +0.60.1 (2022-01-20) +------------------- + +- Fixed a bug where installation on Python 3.6 stopped working due to a change to an underlying dependency. This release can now be installed on Python 3.6, but is the last release of Datasette that will support anything less than Python 3.7. (:issue:`1609`) + .. _v0_60: 0.60 (2022-01-13) From 5bfd001b55357106dba090c83a1c88912a004665 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Feb 2022 15:42:37 -0800 Subject: [PATCH 0616/1866] Use de-dupe idiom that works with Python 3.6, refs #1632 --- datasette/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/cli.py b/datasette/cli.py index 61e7ce91..a8da0741 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -550,7 +550,9 @@ def serve( ) # De-duplicate files so 'datasette db.db db.db' only attaches one /db - files = list(dict.fromkeys(files)) + files_seen = set() + deduped_files = [f for f in files if f not in files_seen and not files_seen.add(f)] + files = deduped_files try: ds = Datasette(files, **kwargs) From 1b2f0ab6bbc9274dac1ba5fe126b1d6b8587ea96 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Feb 2022 15:43:20 -0800 Subject: [PATCH 0617/1866] Revert "Use de-dupe idiom that works with Python 3.6, refs #1632" This reverts commit 5bfd001b55357106dba090c83a1c88912a004665. No need for this on the main branch because it doesn't support Python 3.6 any more. --- datasette/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index a8da0741..61e7ce91 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -550,9 +550,7 @@ def serve( ) # De-duplicate files so 'datasette db.db db.db' only attaches one /db - files_seen = set() - deduped_files = [f for f in files if f not in files_seen and not files_seen.add(f)] - files = deduped_files + files = list(dict.fromkeys(files)) try: ds = Datasette(files, **kwargs) From 458f03ad3a454d271f47a643f4530bd8b60ddb76 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 8 Feb 2022 22:32:19 -0800 Subject: [PATCH 0618/1866] More SpatiaLite details on /-/versions, closes #1607 --- datasette/app.py | 12 ++++++++++++ datasette/utils/__init__.py | 32 ++++++++++++++++++++++++++++++++ tests/test_spatialite.py | 21 +++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 tests/test_spatialite.py diff --git a/datasette/app.py b/datasette/app.py index 7bdf076c..8c5480cf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -46,6 +46,7 @@ from .database import Database, QueryInterrupted from .utils import ( PrefixedUrlString, + SPATIALITE_FUNCTIONS, StartupError, add_cors_headers, async_call_with_supported_arguments, @@ -724,6 +725,17 @@ class Datasette: sqlite_extensions[extension] = None except Exception: pass + # More details on SpatiaLite + if "spatialite" in sqlite_extensions: + spatialite_details = {} + for fn in SPATIALITE_FUNCTIONS: + try: + result = conn.execute("select {}()".format(fn)) + spatialite_details[fn] = result.fetchone()[0] + except Exception as e: + spatialite_details[fn] = {"error": str(e)} + sqlite_extensions["spatialite"] = spatialite_details + # Figure out supported FTS versions fts_versions = [] for fts in ("FTS5", "FTS4", "FTS3"): diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 610e916f..e17b4d7f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -52,9 +52,41 @@ SPATIALITE_PATHS = ( "/usr/local/lib/mod_spatialite.dylib", "/usr/local/lib/mod_spatialite.so", ) +# Used to display /-/versions.json SpatiaLite information +SPATIALITE_FUNCTIONS = ( + "spatialite_version", + "spatialite_target_cpu", + "check_strict_sql_quoting", + "freexl_version", + "proj_version", + "geos_version", + "rttopo_version", + "libxml2_version", + "HasIconv", + "HasMathSQL", + "HasGeoCallbacks", + "HasProj", + "HasProj6", + "HasGeos", + "HasGeosAdvanced", + "HasGeosTrunk", + "HasGeosReentrant", + "HasGeosOnlyReentrant", + "HasMiniZip", + "HasRtTopo", + "HasLibXML2", + "HasEpsg", + "HasFreeXL", + "HasGeoPackage", + "HasGCP", + "HasTopology", + "HasKNN", + "HasRouting", +) # Length of hash subset used in hashed URLs: HASH_LENGTH = 7 + # Can replace this with Column from sqlite_utils when I add that dependency Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py new file mode 100644 index 00000000..8b98c5d6 --- /dev/null +++ b/tests/test_spatialite.py @@ -0,0 +1,21 @@ +from datasette.app import Datasette +from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS +import pytest + + +def has_spatialite(): + try: + find_spatialite() + return True + except SpatialiteNotFound: + return False + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite") +async def test_spatialite_version_info(): + ds = Datasette(sqlite_extensions=["spatialite"]) + response = await ds.client.get("/-/versions.json") + assert response.status_code == 200 + spatialite = response.json()["sqlite"]["extensions"]["spatialite"] + assert set(SPATIALITE_FUNCTIONS) == set(spatialite) From 7d24fd405f3c60e4c852c5d746c91aa2ba23cf5b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 9 Feb 2022 09:47:54 -0800 Subject: [PATCH 0619/1866] datasette-auth-passwords is now an example of register_commands Refs https://github.com/simonw/datasette-auth-passwords/issues/19 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index a63d441e..92cf662f 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -633,7 +633,7 @@ Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-d pip install -e path/to/my/datasette-plugin -Example: `datasette-verify <https://datasette.io/plugins/datasette-verify>`_ +Examples: `datasette-auth-passwords <https://datasette.io/plugins/datasette-auth-passwords>`__, `datasette-verify <https://datasette.io/plugins/datasette-verify>`__ .. _plugin_register_facet_classes: From dd94157f8958bdfe9f45575add934ccf1aba6d63 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 27 Feb 2022 10:04:03 -0800 Subject: [PATCH 0620/1866] Link to tutorials from documentation index page --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index acca943f..a2888822 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover `Explore a demo <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__, watch `a presentation about the project <https://static.simonwillison.net/static/2018/pybay-datasette/>`__ or :ref:`getting_started_glitch`. -More examples: https://datasette.io/examples +Interested in learning Datasette? Start with `the official tutorials <https://datasette.io/tutorials>`__. Support questions, feedback? Join our `GitHub Discussions forum <https://github.com/simonw/datasette/discussions>`__. From 5010d1359b9e9db90a5a69a3ca22d12862893e00 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 5 Mar 2022 11:45:04 -0800 Subject: [PATCH 0621/1866] Fix for test failure caused by SQLite 3.37.0+, closes #1647 --- datasette/templates/_table.html | 2 +- tests/test_internals_database.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index d91a1a57..5332f831 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -4,7 +4,7 @@ <thead> <tr> {% for column in display_columns %} - <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}"> + <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}"> {% if not column.sortable %} {{ column.name }} {% else %} diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index bcecb486..31538a24 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -279,7 +279,15 @@ async def test_table_columns(db, table, expected): @pytest.mark.asyncio async def test_table_column_details(db, table, expected): columns = await db.table_column_details(table) - assert columns == expected + # Convert "type" to lowercase before comparison + # https://github.com/simonw/datasette/issues/1647 + compare_columns = [ + Column( + c.cid, c.name, c.type.lower(), c.notnull, c.default_value, c.is_pk, c.hidden + ) + for c in columns + ] + assert compare_columns == expected @pytest.mark.asyncio From a22ec96c3ac555337eb49121450723a273fb52d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Mar 2022 17:29:53 -0800 Subject: [PATCH 0622/1866] Update pytest-asyncio requirement from <0.17,>=0.10 to >=0.10,<0.19 (#1631) Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.18.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6accaa30..6a097e0f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( "test": [ "pytest>=5.2.2,<6.3.0", "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.10,<0.17", + "pytest-asyncio>=0.10,<0.19", "beautifulsoup4>=4.8.1,<4.11.0", "black==22.1.0", "pytest-timeout>=1.4.2,<2.1", From b21839dd1a005f6269c4e9a9f763195fe7aa9c86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Mar 2022 17:30:05 -0800 Subject: [PATCH 0623/1866] Update pytest requirement from <6.3.0,>=5.2.2 to >=5.2.2,<7.1.0 (#1629) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...7.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6a097e0f..b9db0700 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "test": [ - "pytest>=5.2.2,<6.3.0", + "pytest>=5.2.2,<7.1.0", "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.10,<0.19", "beautifulsoup4>=4.8.1,<4.11.0", From 73f2d25f70d741c6b53f7312674c91f0aec83e17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Mar 2022 17:30:27 -0800 Subject: [PATCH 0624/1866] Update asgiref requirement from <3.5.0,>=3.2.10 to >=3.2.10,<3.6.0 (#1610) Updates the requirements on [asgiref](https://github.com/django/asgiref) to permit the latest version. - [Release notes](https://github.com/django/asgiref/releases) - [Changelog](https://github.com/django/asgiref/blob/main/CHANGELOG.txt) - [Commits](https://github.com/django/asgiref/compare/3.2.10...3.5.0) --- updated-dependencies: - dependency-name: asgiref dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9db0700..b13f7496 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( include_package_data=True, python_requires=">=3.7", install_requires=[ - "asgiref>=3.2.10,<3.5.0", + "asgiref>=3.2.10,<3.6.0", "click>=7.1.1,<8.1.0", "click-default-group~=1.2.2", "Jinja2>=2.10.3,<3.1.0", From 7b78279b93b6e7a5fce6b53e5a85ca421a801496 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Mar 2022 17:41:49 -0800 Subject: [PATCH 0625/1866] Update pytest-timeout requirement from <2.1,>=1.4.2 to >=1.4.2,<2.2 (#1602) Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-timeout/releases) - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/1.4.2...2.1.0) --- updated-dependencies: - dependency-name: pytest-timeout dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b13f7496..8e69c2f5 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( "pytest-asyncio>=0.10,<0.19", "beautifulsoup4>=4.8.1,<4.11.0", "black==22.1.0", - "pytest-timeout>=1.4.2,<2.1", + "pytest-timeout>=1.4.2,<2.2", "trustme>=0.7,<0.10", "cogapp>=3.3.0", ], From 0499f174c063283aa9b589d475a32077aaf7adc5 Mon Sep 17 00:00:00 2001 From: David Larlet <3556+davidbgk@users.noreply.github.com> Date: Sat, 5 Mar 2022 20:58:31 -0500 Subject: [PATCH 0626/1866] Typo in docs about default redirect status code (#1589) --- docs/custom_templates.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 3e4eb633..97dea2af 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -428,7 +428,7 @@ You can use the ``custom_redirect(location)`` function to redirect users to anot Now requests to ``http://localhost:8001/datasette`` will result in a redirect. -These redirects are served with a ``301 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function: +These redirects are served with a ``302 Found`` status code by default. You can send a ``301 Moved Permanently`` code by passing ``301`` as the second argument to the function: .. code-block:: jinja From de810f49cc57a4f88e4a1553d26c579253ce4531 Mon Sep 17 00:00:00 2001 From: Dan Peterson <danp@danp.net> Date: Sun, 6 Mar 2022 15:39:15 -0400 Subject: [PATCH 0627/1866] Add /opt/homebrew to where spatialite extension can be found (#1649) Helps homebrew on Apple Silicon setups find spatialite without needing a full path. Similar to #1114 Thanks, @danp --- datasette/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index e17b4d7f..133b9bc7 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -51,6 +51,7 @@ SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", "/usr/local/lib/mod_spatialite.dylib", "/usr/local/lib/mod_spatialite.so", + "/opt/homebrew/lib/mod_spatialite.dylib", ) # Used to display /-/versions.json SpatiaLite information SPATIALITE_FUNCTIONS = ( From 1baa030eca375f839f3471237547ab403523e643 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 07:38:29 -0800 Subject: [PATCH 0628/1866] Switch to dash encoding for table/database/row-pk in paths * Dash encoding functions, tests and docs, refs #1439 * dash encoding is now like percent encoding but with dashes * Use dash-encoding for row PKs and ?_next=, refs #1439 * Use dash encoding for table names, refs #1439 * Use dash encoding for database names, too, refs #1439 See also https://simonwillison.net/2022/Mar/5/dash-encoding/ --- datasette/url_builder.py | 10 ++++---- datasette/utils/__init__.py | 41 ++++++++++++++++++++++++++--- datasette/views/base.py | 24 ++++++++--------- datasette/views/table.py | 9 ++++--- docs/internals.rst | 26 +++++++++++++++++++ tests/fixtures.py | 1 + tests/test_api.py | 19 +++++++++++--- tests/test_cli.py | 5 ++-- tests/test_html.py | 50 ++++++++++++++++++++++++------------ tests/test_internals_urls.py | 2 +- tests/test_table_api.py | 7 +++-- tests/test_table_html.py | 12 ++++++--- tests/test_utils.py | 20 ++++++++++++++- 13 files changed, 173 insertions(+), 53 deletions(-) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 2bcda869..eebfe31e 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import path_with_format, HASH_LENGTH, PrefixedUrlString +from .utils import dash_encode, path_with_format, HASH_LENGTH, PrefixedUrlString import urllib @@ -31,20 +31,20 @@ class Urls: db = self.ds.databases[database] if self.ds.setting("hash_urls") and db.hash: path = self.path( - f"{urllib.parse.quote(database)}-{db.hash[:HASH_LENGTH]}", format=format + f"{dash_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format ) else: - path = self.path(urllib.parse.quote(database), format=format) + path = self.path(dash_encode(database), format=format) return path def table(self, database, table, format=None): - path = f"{self.database(database)}/{urllib.parse.quote_plus(table)}" + path = f"{self.database(database)}/{dash_encode(table)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) def query(self, database, query, format=None): - path = f"{self.database(database)}/{urllib.parse.quote_plus(query)}" + path = f"{self.database(database)}/{dash_encode(query)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 133b9bc7..79feeef6 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -112,12 +112,12 @@ async def await_me_maybe(value: typing.Any) -> typing.Any: def urlsafe_components(token): - """Splits token on commas and URL decodes each component""" - return [urllib.parse.unquote_plus(b) for b in token.split(",")] + """Splits token on commas and dash-decodes each component""" + return [dash_decode(b) for b in token.split(",")] def path_from_row_pks(row, pks, use_rowid, quote=True): - """Generate an optionally URL-quoted unique identifier + """Generate an optionally dash-quoted unique identifier for a row from its primary keys.""" if use_rowid: bits = [row["rowid"]] @@ -126,7 +126,7 @@ def path_from_row_pks(row, pks, use_rowid, quote=True): row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks ] if quote: - bits = [urllib.parse.quote_plus(str(bit)) for bit in bits] + bits = [dash_encode(str(bit)) for bit in bits] else: bits = [str(bit) for bit in bits] @@ -1140,3 +1140,36 @@ def add_cors_headers(headers): headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Allow-Headers"] = "Authorization" headers["Access-Control-Expose-Headers"] = "Link" + + +_DASH_ENCODING_SAFE = frozenset( + b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" + b"abcdefghijklmnopqrstuvwxyz" + b"0123456789_" + # This is the same as Python percent-encoding but I removed + # '.' and '-' and '~' +) + + +class DashEncoder(dict): + # Keeps a cache internally, via __missing__ + def __missing__(self, b): + # Handle a cache miss, store encoded string in cache and return. + res = chr(b) if b in _DASH_ENCODING_SAFE else "-{:02X}".format(b) + self[b] = res + return res + + +_dash_encoder = DashEncoder().__getitem__ + + +@documented +def dash_encode(s: str) -> str: + "Returns dash-encoded string - for example ``/foo/bar`` -> ``-2Ffoo-2Fbar``" + return "".join(_dash_encoder(char) for char in s.encode("utf-8")) + + +@documented +def dash_decode(s: str) -> str: + "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``" + return urllib.parse.unquote(s.replace("-", "%")) diff --git a/datasette/views/base.py b/datasette/views/base.py index c74d6141..7cd385b7 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -17,6 +17,8 @@ from datasette.utils import ( InvalidSql, LimitedWriter, call_with_supported_arguments, + dash_decode, + dash_encode, path_from_row_pks, path_with_added_args, path_with_removed_args, @@ -203,17 +205,17 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None - db_name = urllib.parse.unquote_plus(db_name) - if db_name not in self.ds.databases and "-" in db_name: + decoded_name = dash_decode(db_name) + if decoded_name not in self.ds.databases and "-" in db_name: # No matching DB found, maybe it's a name-hash? name_bit, hash_bit = db_name.rsplit("-", 1) - if name_bit not in self.ds.databases: + if dash_decode(name_bit) not in self.ds.databases: raise NotFound(f"Database not found: {name}") else: - name = name_bit + name = dash_decode(name_bit) hash = hash_bit else: - name = db_name + name = decoded_name try: db = self.ds.databases[name] @@ -233,9 +235,7 @@ class DataView(BaseView): return await db.table_exists(t) table, _format = await resolve_table_and_format( - table_and_format=urllib.parse.unquote_plus( - kwargs["table_and_format"] - ), + table_and_format=dash_decode(kwargs["table_and_format"]), table_exists=async_table_exists, allowed_formats=self.ds.renderers.keys(), ) @@ -243,11 +243,11 @@ class DataView(BaseView): if _format: kwargs["as_format"] = f".{_format}" elif kwargs.get("table"): - kwargs["table"] = urllib.parse.unquote_plus(kwargs["table"]) + kwargs["table"] = dash_decode(kwargs["table"]) should_redirect = self.ds.urls.path(f"{name}-{expected}") if kwargs.get("table"): - should_redirect += "/" + urllib.parse.quote_plus(kwargs["table"]) + should_redirect += "/" + dash_encode(kwargs["table"]) if kwargs.get("pk_path"): should_redirect += "/" + kwargs["pk_path"] if kwargs.get("as_format"): @@ -467,7 +467,7 @@ class DataView(BaseView): return await db.table_exists(t) table, _ext_format = await resolve_table_and_format( - table_and_format=urllib.parse.unquote_plus(args["table_and_format"]), + table_and_format=dash_decode(args["table_and_format"]), table_exists=async_table_exists, allowed_formats=self.ds.renderers.keys(), ) @@ -475,7 +475,7 @@ class DataView(BaseView): args["table"] = table del args["table_and_format"] elif "table" in args: - args["table"] = urllib.parse.unquote_plus(args["table"]) + args["table"] = dash_decode(args["table"]) return _format, args async def view_get(self, request, database, hash, correct_hash_provided, **kwargs): diff --git a/datasette/views/table.py b/datasette/views/table.py index be9e9c3b..1d81755e 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -12,6 +12,7 @@ from datasette.utils import ( MultiParams, append_querystring, compound_keys_after_sql, + dash_encode, escape_sqlite, filters_should_redirect, is_url, @@ -142,7 +143,7 @@ class RowTableShared(DataView): '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format( base_url=base_url, database=database, - table=urllib.parse.quote_plus(table), + table=dash_encode(table), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) @@ -199,8 +200,8 @@ class RowTableShared(DataView): link_template.format( database=database, base_url=base_url, - table=urllib.parse.quote_plus(other_table), - link_id=urllib.parse.quote_plus(str(value)), + table=dash_encode(other_table), + link_id=dash_encode(str(value)), id=str(markupsafe.escape(value)), label=str(markupsafe.escape(label)) or "-", ) @@ -765,7 +766,7 @@ class TableView(RowTableShared): if prefix is None: prefix = "$null" else: - prefix = urllib.parse.quote_plus(str(prefix)) + prefix = dash_encode(str(prefix)) next_value = f"{prefix},{next_value}" added_args = {"_next": next_value} if sort: diff --git a/docs/internals.rst b/docs/internals.rst index 12ef5c54..d035e1f1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -876,6 +876,32 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth .. autofunction:: datasette.utils.await_me_maybe +.. _internals_dash_encoding: + +Dash encoding +------------- + +Datasette uses a custom encoding scheme in some places, called **dash encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them. + +Dash encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``-`` hyphen character used in place of ``%``. + +Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_`` will be replaced by the numeric equivalent preceded by a hyphen. For example: + +- ``/`` becomes ``-2F`` +- ``.`` becomes ``-2E`` +- ``%`` becomes ``-25`` +- ``-`` becomes ``-2D`` +- Space character becomes ``-20`` +- ``polls/2022.primary`` becomes ``polls-2F2022-2Eprimary`` + +.. _internals_utils_dash_encode: + +.. autofunction:: datasette.utils.dash_encode + +.. _internals_utils_dash_decode: + +.. autofunction:: datasette.utils.dash_decode + .. _internals_tracer: datasette.tracer diff --git a/tests/fixtures.py b/tests/fixtures.py index 26f0cf7b..11f09c41 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -406,6 +406,7 @@ CREATE TABLE compound_primary_key ( ); INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); +INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); CREATE TABLE compound_three_primary_keys ( pk1 varchar(30), diff --git a/tests/test_api.py b/tests/test_api.py index 57471af2..dd916cf0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -143,7 +143,7 @@ def test_database_page(app_client): "name": "compound_primary_key", "columns": ["pk1", "pk2", "content"], "primary_keys": ["pk1", "pk2"], - "count": 1, + "count": 2, "hidden": False, "fts_table": None, "foreign_keys": {"incoming": [], "outgoing": []}, @@ -942,7 +942,7 @@ def test_cors(app_client_with_cors, path, status_code): ) def test_database_with_space_in_name(app_client_two_attached_databases, path): response = app_client_two_attached_databases.get( - "/extra database" + path, follow_redirects=True + "/extra-20database" + path, follow_redirects=True ) assert response.status == 200 @@ -953,7 +953,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): d["name"] for d in app_client_conflicting_database_names.get("/-/databases.json").json ] - for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): + for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-2Dbar.json")): data = app_client_conflicting_database_names.get(path).json assert db_name == data["database"] @@ -992,3 +992,16 @@ async def test_hidden_sqlite_stat1_table(): data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] assert tables == [("normal", False), ("sqlite_stat1", True)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d")) +async def test_dash_encoded_database_names(db_name): + ds = Datasette() + ds.add_memory_database(db_name) + response = await ds.client.get("/.json") + assert db_name in response.json().keys() + path = response.json()[db_name]["path"] + # And the JSON for that database + response2 = await ds.client.get(path + ".json") + assert response2.status_code == 200 diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fbfdee2..e30c2ad3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,6 +9,7 @@ from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ +from datasette.utils import dash_encode from datasette.utils.sqlite import sqlite3 from click.testing import CliRunner import io @@ -294,12 +295,12 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename): assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] expected_link = '<a href="/{}">{}</a>'.format( - urllib.parse.quote(filename_no_stem), filename_no_stem + dash_encode(filename_no_stem), filename_no_stem ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))] + cli, [db_path, "--get", "/{}".format(dash_encode(filename_no_stem))] ) assert result2.exit_code == 0, result2.output diff --git a/tests/test_html.py b/tests/test_html.py index d5f4250d..b4a12b8a 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -29,7 +29,7 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": r"/extra%20database", "text": "extra database"}, + {"href": r"/extra-20database", "text": "extra database"}, {"href": "/fixtures", "text": "fixtures"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # Database should show count text and attached tables @@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": r"/extra%20database/searchable", "text": "searchable"}, - {"href": r"/extra%20database/searchable_view", "text": "searchable_view"}, + {"href": r"/extra-20database/searchable", "text": "searchable"}, + {"href": r"/extra-20database/searchable_view", "text": "searchable_view"}, ] == table_links @@ -140,7 +140,7 @@ def test_database_page(app_client): assert queries_ul is not None assert [ ( - "/fixtures/%F0%9D%90%9C%F0%9D%90%A2%F0%9D%90%AD%F0%9D%90%A2%F0%9D%90%9E%F0%9D%90%AC", + "/fixtures/-F0-9D-90-9C-F0-9D-90-A2-F0-9D-90-AD-F0-9D-90-A2-F0-9D-90-9E-F0-9D-90-AC", "𝐜𝐢𝐭𝐢𝐞𝐬", ), ("/fixtures/from_async_hook", "from_async_hook"), @@ -193,11 +193,11 @@ def test_row_redirects_with_url_hash(app_client_with_hash): def test_row_strange_table_name_with_url_hash(app_client_with_hash): - response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3") + response = app_client_with_hash.get("/fixtures/table-2Fwith-2Fslashes-2Ecsv/3") assert response.status == 302 - assert response.headers["Location"].endswith("/table%2Fwith%2Fslashes.csv/3") + assert response.headers["Location"].endswith("/table-2Fwith-2Fslashes-2Ecsv/3") response = app_client_with_hash.get( - "/fixtures/table%2Fwith%2Fslashes.csv/3", follow_redirects=True + "/fixtures/table-2Fwith-2Fslashes-2Ecsv/3", follow_redirects=True ) assert response.status == 200 @@ -345,20 +345,38 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l assert link == expected_link -def test_row_html_compound_primary_key(app_client): - response = app_client.get("/fixtures/compound_primary_key/a,b") +@pytest.mark.parametrize( + "path,expected", + ( + ( + "/fixtures/compound_primary_key/a,b", + [ + [ + '<td class="col-pk1 type-str">a</td>', + '<td class="col-pk2 type-str">b</td>', + '<td class="col-content type-str">c</td>', + ] + ], + ), + ( + "/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd", + [ + [ + '<td class="col-pk1 type-str">a/b</td>', + '<td class="col-pk2 type-str">.c-d</td>', + '<td class="col-content type-str">c</td>', + ] + ], + ), + ), +) +def test_row_html_compound_primary_key(app_client, path, expected): + response = app_client.get(path) assert response.status == 200 table = Soup(response.body, "html.parser").find("table") assert ["pk1", "pk2", "content"] == [ th.string.strip() for th in table.select("thead th") ] - expected = [ - [ - '<td class="col-pk1 type-str">a</td>', - '<td class="col-pk2 type-str">b</td>', - '<td class="col-content type-str">c</td>', - ] - ] assert expected == [ [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") ] diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index e486e4c9..16515ad6 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -121,7 +121,7 @@ def test_database(ds, base_url, format, expected): ("/", "name", None, "/_memory/name"), ("/prefix/", "name", None, "/prefix/_memory/name"), ("/", "name", "json", "/_memory/name.json"), - ("/", "name.json", "json", "/_memory/name.json?_format=json"), + ("/", "name.json", "json", "/_memory/name-2Ejson.json"), ], ) def test_table_and_query(ds, base_url, name, format, expected): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 6a6daed5..cc38d392 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -136,7 +136,10 @@ def test_table_shape_object(app_client): def test_table_shape_object_compound_primary_key(app_client): response = app_client.get("/fixtures/compound_primary_key.json?_shape=object") - assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json + assert response.json == { + "a,b": {"pk1": "a", "pk2": "b", "content": "c"}, + "a-2Fb,-2Ec-2Dd": {"pk1": "a/b", "pk2": ".c-d", "content": "c"}, + } def test_table_with_slashes_in_name(app_client): @@ -308,7 +311,7 @@ def test_sortable(app_client, query_string, sort_key, human_description_en): path = response.json["next_url"] if path: path = path.replace("http://localhost", "") - assert 5 == page + assert page == 5 expected = list(generate_sortable_rows(201)) expected.sort(key=sort_key) assert [r["content"] for r in expected] == [r["content"] for r in fetched] diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 021268c3..77d97d80 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -563,11 +563,17 @@ def test_table_html_compound_primary_key(app_client): '<td class="col-pk1 type-str">a</td>', '<td class="col-pk2 type-str">b</td>', '<td class="col-content type-str">c</td>', - ] + ], + [ + '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd">a/b,.c-d</a></td>', + '<td class="col-pk1 type-str">a/b</td>', + '<td class="col-pk2 type-str">.c-d</td>', + '<td class="col-content type-str">c</td>', + ], ] - assert expected == [ + assert [ [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] + ] == expected def test_table_html_foreign_key_links(app_client): diff --git a/tests/test_utils.py b/tests/test_utils.py index e7d67045..1c3ab495 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -93,7 +93,7 @@ def test_path_with_replaced_args(path, args, expected): "row,pks,expected_path", [ ({"A": "foo", "B": "bar"}, ["A", "B"], "foo,bar"), - ({"A": "f,o", "B": "bar"}, ["A", "B"], "f%2Co,bar"), + ({"A": "f,o", "B": "bar"}, ["A", "B"], "f-2Co,bar"), ({"A": 123}, ["A"], "123"), ( utils.CustomRow( @@ -646,3 +646,21 @@ async def test_derive_named_parameters(sql, expected): db = ds.get_database("_memory") params = await utils.derive_named_parameters(db, sql) assert params == expected + + +@pytest.mark.parametrize( + "original,expected", + ( + ("abc", "abc"), + ("/foo/bar", "-2Ffoo-2Fbar"), + ("/-/bar", "-2F-2D-2Fbar"), + ("-/db-/table.csv", "-2D-2Fdb-2D-2Ftable-2Ecsv"), + (r"%~-/", "-25-7E-2D-2F"), + ("-25-7E-2D-2F", "-2D25-2D7E-2D2D-2D2F"), + ), +) +def test_dash_encoding(original, expected): + actual = utils.dash_encode(original) + assert actual == expected + # And test round-trip + assert original == utils.dash_decode(actual) From 644d25d1de78a36b105cca479e7b3e4375a6eadc Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 08:01:03 -0800 Subject: [PATCH 0629/1866] Redirect old % URLs to new - encoded URLs, closes #1650 Refs #1439 --- datasette/app.py | 7 +++++++ tests/test_html.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 8c5480cf..2907d90e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1211,6 +1211,13 @@ class DatasetteRouter: return await self.handle_404(request, send) async def handle_404(self, request, send, exception=None): + # If path contains % encoding, redirect to dash encoding + if "%" in request.path: + # Try the same path but with "%" replaced by "-" + # and "-" replaced with "-2D" + new_path = request.path.replace("-", "-2D").replace("%", "-") + await asgi_send_redirect(send, new_path) + return # If URL has a trailing slash, redirect to URL without it path = request.scope.get( "raw_path", request.scope["path"].encode("utf8") diff --git a/tests/test_html.py b/tests/test_html.py index b4a12b8a..3e24009e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -952,3 +952,9 @@ def test_no_alternate_url_json(app_client, path): assert ( '<link rel="alternate" type="application/json+datasette"' not in response.text ) + + +def test_redirect_percent_encoding_to_dash_encoding(app_client): + response = app_client.get("/fivethirtyeight/twitter-ratio%2Fsenators") + assert response.status == 302 + assert response.headers["location"] == "/fivethirtyeight/twitter-2Dratio-2Fsenators" From d714c67d656c46e012b24ccca53b59409440334f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 08:09:15 -0800 Subject: [PATCH 0630/1866] asyncio_mode = strict to avoid pytest warnings --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index d702ce5f..559e518c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,3 +8,4 @@ filterwarnings= ignore:.*current_task.*:PendingDeprecationWarning markers = serial: tests to avoid using with pytest-xdist +asyncio_mode = strict From 020effe47bf89f35182960a9645f2383a42ebd54 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 08:18:07 -0800 Subject: [PATCH 0631/1866] Preserve query string in % to - redirects, refs #1650 --- datasette/app.py | 2 ++ tests/test_html.py | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 2907d90e..7abccc05 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1216,6 +1216,8 @@ class DatasetteRouter: # Try the same path but with "%" replaced by "-" # and "-" replaced with "-2D" new_path = request.path.replace("-", "-2D").replace("%", "-") + if request.query_string: + new_path += "?{}".format(request.query_string) await asgi_send_redirect(send, new_path) return # If URL has a trailing slash, redirect to URL without it diff --git a/tests/test_html.py b/tests/test_html.py index 3e24009e..de703284 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -954,7 +954,18 @@ def test_no_alternate_url_json(app_client, path): ) -def test_redirect_percent_encoding_to_dash_encoding(app_client): - response = app_client.get("/fivethirtyeight/twitter-ratio%2Fsenators") +@pytest.mark.parametrize( + "path,expected", + ( + ( + "/fivethirtyeight/twitter-ratio%2Fsenators", + "/fivethirtyeight/twitter-2Dratio-2Fsenators", + ), + # query string should be preserved + ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"), + ), +) +def test_redirect_percent_encoding_to_dash_encoding(app_client, path, expected): + response = app_client.get(path) assert response.status == 302 - assert response.headers["location"] == "/fivethirtyeight/twitter-2Dratio-2Fsenators" + assert response.headers["location"] == expected From c85d669de387b40e667fd6942c6cc1c15b4f5964 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 11:26:08 -0800 Subject: [PATCH 0632/1866] Fix bug with percentage redirects, close #1650 --- datasette/utils/__init__.py | 7 ++++++- tests/test_html.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 79feeef6..e7c9fb1c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -10,6 +10,7 @@ import markupsafe import mergedeep import os import re +import secrets import shlex import tempfile import typing @@ -1172,4 +1173,8 @@ def dash_encode(s: str) -> str: @documented def dash_decode(s: str) -> str: "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``" - return urllib.parse.unquote(s.replace("-", "%")) + # Avoid accidentally decoding a %2f style sequence + temp = secrets.token_hex(16) + s = s.replace("%", temp) + decoded = urllib.parse.unquote(s.replace("-", "%")) + return decoded.replace(temp, "%") diff --git a/tests/test_html.py b/tests/test_html.py index de703284..55d78c05 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -961,6 +961,10 @@ def test_no_alternate_url_json(app_client, path): "/fivethirtyeight/twitter-ratio%2Fsenators", "/fivethirtyeight/twitter-2Dratio-2Fsenators", ), + ( + "/fixtures/table%2Fwith%2Fslashes", + "/fixtures/table-2Fwith-2Fslashes", + ), # query string should be preserved ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"), ), From bb499942c15c4e2cfa4b6afab8f8debe5948c009 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 11:33:31 -0800 Subject: [PATCH 0633/1866] Fixed tests for urlsafe_components, refs #1650 --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1c3ab495..ff4f649a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,8 +19,8 @@ from unittest.mock import patch ("foo", ["foo"]), ("foo,bar", ["foo", "bar"]), ("123,433,112", ["123", "433", "112"]), - ("123%2C433,112", ["123,433", "112"]), - ("123%2F433%2F112", ["123/433/112"]), + ("123-2C433,112", ["123,433", "112"]), + ("123-2F433-2F112", ["123/433/112"]), ], ) def test_urlsafe_components(path, expected): From c5791156d92615f25696ba93dae5bb2dcc192c98 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 14:04:10 -0800 Subject: [PATCH 0634/1866] Code of conduct, refs #1654 --- CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..14d4c567 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +`swillison+datasette-code-of-conduct@gmail.com`. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. From 239aed182053903ed69108776b6864d42bfe1eb4 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 15 Mar 2022 08:36:35 -0700 Subject: [PATCH 0635/1866] Revert "Code of conduct, refs #1654" This reverts commit c5791156d92615f25696ba93dae5bb2dcc192c98. Refs #1658 --- CODE_OF_CONDUCT.md | 128 --------------------------------------------- 1 file changed, 128 deletions(-) delete mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 14d4c567..00000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,128 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -`swillison+datasette-code-of-conduct@gmail.com`. -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series -of actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within -the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. From 5a353a32b9c4d75acbe3193fd72f735a8e78516a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 15 Mar 2022 08:37:14 -0700 Subject: [PATCH 0636/1866] Revert "Fixed tests for urlsafe_components, refs #1650" This reverts commit bb499942c15c4e2cfa4b6afab8f8debe5948c009. Refs #1658 --- tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index ff4f649a..1c3ab495 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,8 +19,8 @@ from unittest.mock import patch ("foo", ["foo"]), ("foo,bar", ["foo", "bar"]), ("123,433,112", ["123", "433", "112"]), - ("123-2C433,112", ["123,433", "112"]), - ("123-2F433-2F112", ["123/433/112"]), + ("123%2C433,112", ["123,433", "112"]), + ("123%2F433%2F112", ["123/433/112"]), ], ) def test_urlsafe_components(path, expected): From 77e718c3ffb30473759a8b1ed347f73cb2ff5cfe Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 15 Mar 2022 08:37:31 -0700 Subject: [PATCH 0637/1866] Revert "Fix bug with percentage redirects, close #1650" This reverts commit c85d669de387b40e667fd6942c6cc1c15b4f5964. Refs #1658 --- datasette/utils/__init__.py | 7 +------ tests/test_html.py | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index e7c9fb1c..79feeef6 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -10,7 +10,6 @@ import markupsafe import mergedeep import os import re -import secrets import shlex import tempfile import typing @@ -1173,8 +1172,4 @@ def dash_encode(s: str) -> str: @documented def dash_decode(s: str) -> str: "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``" - # Avoid accidentally decoding a %2f style sequence - temp = secrets.token_hex(16) - s = s.replace("%", temp) - decoded = urllib.parse.unquote(s.replace("-", "%")) - return decoded.replace(temp, "%") + return urllib.parse.unquote(s.replace("-", "%")) diff --git a/tests/test_html.py b/tests/test_html.py index 55d78c05..de703284 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -961,10 +961,6 @@ def test_no_alternate_url_json(app_client, path): "/fivethirtyeight/twitter-ratio%2Fsenators", "/fivethirtyeight/twitter-2Dratio-2Fsenators", ), - ( - "/fixtures/table%2Fwith%2Fslashes", - "/fixtures/table-2Fwith-2Fslashes", - ), # query string should be preserved ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"), ), From 645381a5ed23c016281e8c6c7d141518f91b67e5 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 15 Mar 2022 08:36:35 -0700 Subject: [PATCH 0638/1866] Add code of conduct again Refs #1658 --- CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..14d4c567 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +`swillison+datasette-code-of-conduct@gmail.com`. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. From c10cd48baf106659bf3f129ad7bfb2226be73821 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 7 Mar 2022 11:56:59 -0800 Subject: [PATCH 0639/1866] Min pytest-asyncio of 0.17 So that the asyncio_mode in pytest.ini does not produce a warning on older versions of that library. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8e69c2f5..e70839d6 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( "test": [ "pytest>=5.2.2,<7.1.0", "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.10,<0.19", + "pytest-asyncio>=0.17,<0.19", "beautifulsoup4>=4.8.1,<4.11.0", "black==22.1.0", "pytest-timeout>=1.4.2,<2.2", From a35393b29cfb5b8abdc6a94e577af1c9a5c13652 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 15 Mar 2022 11:01:57 -0700 Subject: [PATCH 0640/1866] Tilde encoding (#1659) Closes #1657 Refs #1439 --- datasette/app.py | 11 +++++---- datasette/url_builder.py | 10 ++++---- datasette/utils/__init__.py | 37 ++++++++++++++++------------- datasette/views/base.py | 25 +++++++++++--------- datasette/views/table.py | 14 +++++++---- docs/csv_export.rst | 18 --------------- docs/internals.rst | 34 +++++++++++++-------------- tests/test_api.py | 17 ++++---------- tests/test_cli.py | 6 ++--- tests/test_html.py | 45 +++++++++++++++++++++--------------- tests/test_internals_urls.py | 2 +- tests/test_table_api.py | 9 +++++--- tests/test_table_html.py | 2 +- tests/test_utils.py | 36 +++++++++-------------------- 14 files changed, 125 insertions(+), 141 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 7abccc05..b39ef7cd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1211,11 +1211,14 @@ class DatasetteRouter: return await self.handle_404(request, send) async def handle_404(self, request, send, exception=None): - # If path contains % encoding, redirect to dash encoding + # If path contains % encoding, redirect to tilde encoding if "%" in request.path: - # Try the same path but with "%" replaced by "-" - # and "-" replaced with "-2D" - new_path = request.path.replace("-", "-2D").replace("%", "-") + # Try the same path but with "%" replaced by "~" + # and "~" replaced with "~7E" + # and "." replaced with "~2E" + new_path = ( + request.path.replace("~", "~7E").replace("%", "~").replace(".", "~2E") + ) if request.query_string: new_path += "?{}".format(request.query_string) await asgi_send_redirect(send, new_path) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index eebfe31e..9f072462 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import dash_encode, path_with_format, HASH_LENGTH, PrefixedUrlString +from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString import urllib @@ -31,20 +31,20 @@ class Urls: db = self.ds.databases[database] if self.ds.setting("hash_urls") and db.hash: path = self.path( - f"{dash_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format + f"{tilde_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format ) else: - path = self.path(dash_encode(database), format=format) + path = self.path(tilde_encode(database), format=format) return path def table(self, database, table, format=None): - path = f"{self.database(database)}/{dash_encode(table)}" + path = f"{self.database(database)}/{tilde_encode(table)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) def query(self, database, query, format=None): - path = f"{self.database(database)}/{dash_encode(query)}" + path = f"{self.database(database)}/{tilde_encode(query)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 79feeef6..bd591459 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -15,6 +15,7 @@ import tempfile import typing import time import types +import secrets import shutil import urllib import yaml @@ -112,12 +113,12 @@ async def await_me_maybe(value: typing.Any) -> typing.Any: def urlsafe_components(token): - """Splits token on commas and dash-decodes each component""" - return [dash_decode(b) for b in token.split(",")] + """Splits token on commas and tilde-decodes each component""" + return [tilde_decode(b) for b in token.split(",")] def path_from_row_pks(row, pks, use_rowid, quote=True): - """Generate an optionally dash-quoted unique identifier + """Generate an optionally tilde-encoded unique identifier for a row from its primary keys.""" if use_rowid: bits = [row["rowid"]] @@ -126,7 +127,7 @@ def path_from_row_pks(row, pks, use_rowid, quote=True): row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks ] if quote: - bits = [dash_encode(str(bit)) for bit in bits] + bits = [tilde_encode(str(bit)) for bit in bits] else: bits = [str(bit) for bit in bits] @@ -1142,34 +1143,38 @@ def add_cors_headers(headers): headers["Access-Control-Expose-Headers"] = "Link" -_DASH_ENCODING_SAFE = frozenset( +_TILDE_ENCODING_SAFE = frozenset( b"ABCDEFGHIJKLMNOPQRSTUVWXYZ" b"abcdefghijklmnopqrstuvwxyz" - b"0123456789_" + b"0123456789_-" # This is the same as Python percent-encoding but I removed - # '.' and '-' and '~' + # '.' and '~' ) -class DashEncoder(dict): +class TildeEncoder(dict): # Keeps a cache internally, via __missing__ def __missing__(self, b): # Handle a cache miss, store encoded string in cache and return. - res = chr(b) if b in _DASH_ENCODING_SAFE else "-{:02X}".format(b) + res = chr(b) if b in _TILDE_ENCODING_SAFE else "~{:02X}".format(b) self[b] = res return res -_dash_encoder = DashEncoder().__getitem__ +_tilde_encoder = TildeEncoder().__getitem__ @documented -def dash_encode(s: str) -> str: - "Returns dash-encoded string - for example ``/foo/bar`` -> ``-2Ffoo-2Fbar``" - return "".join(_dash_encoder(char) for char in s.encode("utf-8")) +def tilde_encode(s: str) -> str: + "Returns tilde-encoded string - for example ``/foo/bar`` -> ``~2Ffoo~2Fbar``" + return "".join(_tilde_encoder(char) for char in s.encode("utf-8")) @documented -def dash_decode(s: str) -> str: - "Decodes a dash-encoded string, so ``-2Ffoo-2Fbar`` -> ``/foo/bar``" - return urllib.parse.unquote(s.replace("-", "%")) +def tilde_decode(s: str) -> str: + "Decodes a tilde-encoded string, so ``~2Ffoo~2Fbar`` -> ``/foo/bar``" + # Avoid accidentally decoding a %2f style sequence + temp = secrets.token_hex(16) + s = s.replace("%", temp) + decoded = urllib.parse.unquote(s.replace("~", "%")) + return decoded.replace(temp, "%") diff --git a/datasette/views/base.py b/datasette/views/base.py index 7cd385b7..1c0c3f9b 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -10,6 +10,7 @@ import pint from datasette import __version__ from datasette.database import QueryInterrupted +from datasette.utils.asgi import Request from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -17,8 +18,8 @@ from datasette.utils import ( InvalidSql, LimitedWriter, call_with_supported_arguments, - dash_decode, - dash_encode, + tilde_decode, + tilde_encode, path_from_row_pks, path_with_added_args, path_with_removed_args, @@ -205,14 +206,14 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None - decoded_name = dash_decode(db_name) + decoded_name = tilde_decode(db_name) if decoded_name not in self.ds.databases and "-" in db_name: # No matching DB found, maybe it's a name-hash? name_bit, hash_bit = db_name.rsplit("-", 1) - if dash_decode(name_bit) not in self.ds.databases: + if tilde_decode(name_bit) not in self.ds.databases: raise NotFound(f"Database not found: {name}") else: - name = dash_decode(name_bit) + name = tilde_decode(name_bit) hash = hash_bit else: name = decoded_name @@ -235,7 +236,7 @@ class DataView(BaseView): return await db.table_exists(t) table, _format = await resolve_table_and_format( - table_and_format=dash_decode(kwargs["table_and_format"]), + table_and_format=tilde_decode(kwargs["table_and_format"]), table_exists=async_table_exists, allowed_formats=self.ds.renderers.keys(), ) @@ -243,11 +244,11 @@ class DataView(BaseView): if _format: kwargs["as_format"] = f".{_format}" elif kwargs.get("table"): - kwargs["table"] = dash_decode(kwargs["table"]) + kwargs["table"] = tilde_decode(kwargs["table"]) should_redirect = self.ds.urls.path(f"{name}-{expected}") if kwargs.get("table"): - should_redirect += "/" + dash_encode(kwargs["table"]) + should_redirect += "/" + tilde_encode(kwargs["table"]) if kwargs.get("pk_path"): should_redirect += "/" + kwargs["pk_path"] if kwargs.get("as_format"): @@ -291,6 +292,7 @@ class DataView(BaseView): if not request.args.get(key) ] if extra_parameters: + # Replace request object with a new one with modified scope if not request.query_string: new_query_string = "&".join(extra_parameters) else: @@ -300,7 +302,8 @@ class DataView(BaseView): new_scope = dict( request.scope, query_string=new_query_string.encode("latin-1") ) - request.scope = new_scope + receive = request.receive + request = Request(new_scope, receive) if stream: # Some quick soundness checks if not self.ds.setting("allow_csv_stream"): @@ -467,7 +470,7 @@ class DataView(BaseView): return await db.table_exists(t) table, _ext_format = await resolve_table_and_format( - table_and_format=dash_decode(args["table_and_format"]), + table_and_format=tilde_decode(args["table_and_format"]), table_exists=async_table_exists, allowed_formats=self.ds.renderers.keys(), ) @@ -475,7 +478,7 @@ class DataView(BaseView): args["table"] = table del args["table_and_format"] elif "table" in args: - args["table"] = dash_decode(args["table"]) + args["table"] = tilde_decode(args["table"]) return _format, args async def view_get(self, request, database, hash, correct_hash_provided, **kwargs): diff --git a/datasette/views/table.py b/datasette/views/table.py index 1d81755e..72b8e9a4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -12,7 +12,8 @@ from datasette.utils import ( MultiParams, append_querystring, compound_keys_after_sql, - dash_encode, + tilde_decode, + tilde_encode, escape_sqlite, filters_should_redirect, is_url, @@ -143,7 +144,7 @@ class RowTableShared(DataView): '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format( base_url=base_url, database=database, - table=dash_encode(table), + table=tilde_encode(table), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) @@ -200,8 +201,8 @@ class RowTableShared(DataView): link_template.format( database=database, base_url=base_url, - table=dash_encode(other_table), - link_id=dash_encode(str(value)), + table=tilde_encode(other_table), + link_id=tilde_encode(str(value)), id=str(markupsafe.escape(value)), label=str(markupsafe.escape(label)) or "-", ) @@ -346,6 +347,8 @@ class TableView(RowTableShared): write=bool(canned_query.get("write")), ) + table = tilde_decode(table) + db = self.ds.databases[database] is_view = bool(await db.get_view_definition(table)) table_exists = bool(await db.table_exists(table)) @@ -766,7 +769,7 @@ class TableView(RowTableShared): if prefix is None: prefix = "$null" else: - prefix = dash_encode(str(prefix)) + prefix = tilde_encode(str(prefix)) next_value = f"{prefix},{next_value}" added_args = {"_next": next_value} if sort: @@ -938,6 +941,7 @@ class RowView(RowTableShared): name = "row" async def data(self, request, database, hash, table, pk_path, default_labels=False): + table = tilde_decode(table) await self.check_permissions( request, [ diff --git a/docs/csv_export.rst b/docs/csv_export.rst index b1cc673c..023fa05e 100644 --- a/docs/csv_export.rst +++ b/docs/csv_export.rst @@ -59,21 +59,3 @@ truncation error message. You can increase or remove this limit using the :ref:`setting_max_csv_mb` config setting. You can also disable the CSV export feature entirely using :ref:`setting_allow_csv_stream`. - -A note on URLs --------------- - -The default URL for the CSV representation of a table is that table with -``.csv`` appended to it: - -* https://latest.datasette.io/fixtures/facetable - HTML interface -* https://latest.datasette.io/fixtures/facetable.csv - CSV export -* https://latest.datasette.io/fixtures/facetable.json - JSON API - -This pattern doesn't work for tables with names that already end in ``.csv`` or -``.json``. For those tables, you can instead use the ``_format=`` query string -parameter: - -* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv - HTML interface -* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=csv - CSV export -* https://latest.datasette.io/fixtures/table%2Fwith%2Fslashes.csv?_format=json - JSON API diff --git a/docs/internals.rst b/docs/internals.rst index d035e1f1..3d223603 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -545,7 +545,7 @@ These functions can be accessed via the ``{{ urls }}`` object in Datasette templ <a href="{{ urls.table("fixtures", "facetable") }}">facetable table</a> <a href="{{ urls.query("fixtures", "pragma_cache_size") }}">pragma_cache_size query</a> -Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is usually the path with ``.json`` added on the end, but it may use ``?_format=json`` in cases where the path already includes ``.json``, for example a URL to a table named ``table.json``. +Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is the path with ``.json`` added on the end. These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path. @@ -876,31 +876,31 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth .. autofunction:: datasette.utils.await_me_maybe -.. _internals_dash_encoding: +.. _internals_tilde_encoding: -Dash encoding -------------- +Tilde encoding +-------------- -Datasette uses a custom encoding scheme in some places, called **dash encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them. +Datasette uses a custom encoding scheme in some places, called **tilde encoding**. This is primarily used for table names and row primary keys, to avoid any confusion between ``/`` characters in those values and the Datasette URLs that reference them. -Dash encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``-`` hyphen character used in place of ``%``. +Tilde encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``~`` tilde character used in place of ``%``. -Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_`` will be replaced by the numeric equivalent preceded by a hyphen. For example: +Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example: -- ``/`` becomes ``-2F`` -- ``.`` becomes ``-2E`` -- ``%`` becomes ``-25`` -- ``-`` becomes ``-2D`` -- Space character becomes ``-20`` -- ``polls/2022.primary`` becomes ``polls-2F2022-2Eprimary`` +- ``/`` becomes ``~2F`` +- ``.`` becomes ``~2E`` +- ``%`` becomes ``~25`` +- ``~`` becomes ``~7E`` +- Space character becomes ``~20`` +- ``polls/2022.primary`` becomes ``polls~2F2022~2Eprimary`` -.. _internals_utils_dash_encode: +.. _internals_utils_tilde_encode: -.. autofunction:: datasette.utils.dash_encode +.. autofunction:: datasette.utils.tilde_encode -.. _internals_utils_dash_decode: +.. _internals_utils_tilde_decode: -.. autofunction:: datasette.utils.dash_decode +.. autofunction:: datasette.utils.tilde_decode .. _internals_tracer: diff --git a/tests/test_api.py b/tests/test_api.py index dd916cf0..87d91e56 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -679,18 +679,9 @@ def test_row(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] -def test_row_format_in_querystring(app_client): - # regression test for https://github.com/simonw/datasette/issues/563 - response = app_client.get( - "/fixtures/simple_primary_key/1?_format=json&_shape=objects" - ) - assert response.status == 200 - assert [{"id": "1", "content": "hello"}] == response.json["rows"] - - def test_row_strange_table_name(app_client): response = app_client.get( - "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" + "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3.json?_shape=objects" ) assert response.status == 200 assert [{"pk": "3", "content": "hey"}] == response.json["rows"] @@ -942,7 +933,7 @@ def test_cors(app_client_with_cors, path, status_code): ) def test_database_with_space_in_name(app_client_two_attached_databases, path): response = app_client_two_attached_databases.get( - "/extra-20database" + path, follow_redirects=True + "/extra~20database" + path, follow_redirects=True ) assert response.status == 200 @@ -953,7 +944,7 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): d["name"] for d in app_client_conflicting_database_names.get("/-/databases.json").json ] - for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-2Dbar.json")): + for db_name, path in (("foo", "/foo.json"), ("foo-bar", "/foo-bar.json")): data = app_client_conflicting_database_names.get(path).json assert db_name == data["database"] @@ -996,7 +987,7 @@ async def test_hidden_sqlite_stat1_table(): @pytest.mark.asyncio @pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d")) -async def test_dash_encoded_database_names(db_name): +async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") diff --git a/tests/test_cli.py b/tests/test_cli.py index e30c2ad3..5afe72c1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ -from datasette.utils import dash_encode +from datasette.utils import tilde_encode from datasette.utils.sqlite import sqlite3 from click.testing import CliRunner import io @@ -295,12 +295,12 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename): assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] expected_link = '<a href="/{}">{}</a>'.format( - dash_encode(filename_no_stem), filename_no_stem + tilde_encode(filename_no_stem), filename_no_stem ) assert expected_link in result1.output # Now try hitting that database page result2 = runner.invoke( - cli, [db_path, "--get", "/{}".format(dash_encode(filename_no_stem))] + cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))] ) assert result2.exit_code == 0, result2.output diff --git a/tests/test_html.py b/tests/test_html.py index de703284..76a8423a 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -29,7 +29,7 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": r"/extra-20database", "text": "extra database"}, + {"href": "/extra~20database", "text": "extra database"}, {"href": "/fixtures", "text": "fixtures"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # Database should show count text and attached tables @@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": r"/extra-20database/searchable", "text": "searchable"}, - {"href": r"/extra-20database/searchable_view", "text": "searchable_view"}, + {"href": r"/extra~20database/searchable", "text": "searchable"}, + {"href": r"/extra~20database/searchable_view", "text": "searchable_view"}, ] == table_links @@ -139,15 +139,15 @@ def test_database_page(app_client): queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ - ( - "/fixtures/-F0-9D-90-9C-F0-9D-90-A2-F0-9D-90-AD-F0-9D-90-A2-F0-9D-90-9E-F0-9D-90-AC", - "𝐜𝐢𝐭𝐢𝐞𝐬", - ), ("/fixtures/from_async_hook", "from_async_hook"), ("/fixtures/from_hook", "from_hook"), ("/fixtures/magic_parameters", "magic_parameters"), ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"), ("/fixtures/pragma_cache_size", "pragma_cache_size"), + ( + "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC", + "𝐜𝐢𝐭𝐢𝐞𝐬", + ), ] == sorted( [(a["href"], a.text) for a in queries_ul.find_all("a")], key=lambda p: p[0] ) @@ -193,11 +193,11 @@ def test_row_redirects_with_url_hash(app_client_with_hash): def test_row_strange_table_name_with_url_hash(app_client_with_hash): - response = app_client_with_hash.get("/fixtures/table-2Fwith-2Fslashes-2Ecsv/3") + response = app_client_with_hash.get("/fixtures/table~2Fwith~2Fslashes~2Ecsv/3") assert response.status == 302 - assert response.headers["Location"].endswith("/table-2Fwith-2Fslashes-2Ecsv/3") + assert response.headers["Location"].endswith("/table~2Fwith~2Fslashes~2Ecsv/3") response = app_client_with_hash.get( - "/fixtures/table-2Fwith-2Fslashes-2Ecsv/3", follow_redirects=True + "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3", follow_redirects=True ) assert response.status == 200 @@ -229,7 +229,7 @@ def test_row_page_does_not_truncate(): ["query", "db-fixtures", "query-neighborhood_search"], ), ( - "/fixtures/table%2Fwith%2Fslashes.csv", + "/fixtures/table~2Fwith~2Fslashes~2Ecsv", ["table", "db-fixtures", "table-tablewithslashescsv-fa7563"], ), ( @@ -255,7 +255,7 @@ def test_css_classes_on_body(app_client, path, expected_classes): "table-fixtures-simple_primary_key.html, *table.html", ), ( - "/fixtures/table%2Fwith%2Fslashes.csv", + "/fixtures/table~2Fwith~2Fslashes~2Ecsv", "table-fixtures-tablewithslashescsv-fa7563.html, *table.html", ), ( @@ -359,7 +359,7 @@ def test_row_links_from_other_tables(app_client, path, expected_text, expected_l ], ), ( - "/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd", + "/fixtures/compound_primary_key/a~2Fb,~2Ec~2Dd", [ [ '<td class="col-pk1 type-str">a/b</td>', @@ -816,7 +816,8 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ), ("/fixtures/pragma_cache_size", None), ( - "/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬", + # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬 + "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC", "/fixtures?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B", ), ("/fixtures/magic_parameters", None), @@ -824,6 +825,7 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ) def test_edit_sql_link_on_canned_queries(app_client, path, expected): response = app_client.get(path) + assert response.status == 200 expected_link = f'<a href="{expected}" class="canned-query-edit-sql">Edit SQL</a>' if expected: assert expected_link in response.text @@ -898,8 +900,8 @@ def test_trace_correctly_escaped(app_client): # Table page ("/fixtures/facetable", "http://localhost/fixtures/facetable.json"), ( - "/fixtures/table%2Fwith%2Fslashes.csv", - "http://localhost/fixtures/table%2Fwith%2Fslashes.csv?_format=json", + "/fixtures/table~2Fwith~2Fslashes~2Ecsv", + "http://localhost/fixtures/table~2Fwith~2Fslashes~2Ecsv.json", ), # Row page ( @@ -930,6 +932,7 @@ def test_trace_correctly_escaped(app_client): ) def test_alternate_url_json(app_client, path, expected): response = app_client.get(path) + assert response.status == 200 link = response.headers["link"] assert link == '{}; rel="alternate"; type="application/json+datasette"'.format( expected @@ -959,13 +962,17 @@ def test_no_alternate_url_json(app_client, path): ( ( "/fivethirtyeight/twitter-ratio%2Fsenators", - "/fivethirtyeight/twitter-2Dratio-2Fsenators", + "/fivethirtyeight/twitter-ratio~2Fsenators", + ), + ( + "/fixtures/table%2Fwith%2Fslashes.csv", + "/fixtures/table~2Fwith~2Fslashes~2Ecsv", ), # query string should be preserved - ("/foo/bar%2Fbaz?id=5", "/foo/bar-2Fbaz?id=5"), + ("/foo/bar%2Fbaz?id=5", "/foo/bar~2Fbaz?id=5"), ), ) -def test_redirect_percent_encoding_to_dash_encoding(app_client, path, expected): +def test_redirect_percent_encoding_to_tilde_encoding(app_client, path, expected): response = app_client.get(path) assert response.status == 302 assert response.headers["location"] == expected diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 16515ad6..4307789c 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -121,7 +121,7 @@ def test_database(ds, base_url, format, expected): ("/", "name", None, "/_memory/name"), ("/prefix/", "name", None, "/prefix/_memory/name"), ("/", "name", "json", "/_memory/name.json"), - ("/", "name.json", "json", "/_memory/name-2Ejson.json"), + ("/", "name.json", "json", "/_memory/name~2Ejson.json"), ], ) def test_table_and_query(ds, base_url, name, format, expected): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cc38d392..3ab369b3 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -138,13 +138,13 @@ def test_table_shape_object_compound_primary_key(app_client): response = app_client.get("/fixtures/compound_primary_key.json?_shape=object") assert response.json == { "a,b": {"pk1": "a", "pk2": "b", "content": "c"}, - "a-2Fb,-2Ec-2Dd": {"pk1": "a/b", "pk2": ".c-d", "content": "c"}, + "a~2Fb,~2Ec-d": {"pk1": "a/b", "pk2": ".c-d", "content": "c"}, } def test_table_with_slashes_in_name(app_client): response = app_client.get( - "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" + "/fixtures/table~2Fwith~2Fslashes~2Ecsv.json?_shape=objects" ) assert response.status == 200 data = response.json @@ -1032,7 +1032,10 @@ def test_infinity_returned_as_invalid_json_if_requested(app_client): def test_custom_query_with_unicode_characters(app_client): - response = app_client.get("/fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json?_shape=array") + # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json + response = app_client.get( + "/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC.json?_shape=array" + ) assert [{"id": 1, "name": "San Francisco"}] == response.json diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 77d97d80..d40f017a 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -565,7 +565,7 @@ def test_table_html_compound_primary_key(app_client): '<td class="col-content type-str">c</td>', ], [ - '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a-2Fb,-2Ec-2Dd">a/b,.c-d</a></td>', + '<td class="col-Link type-pk"><a href="/fixtures/compound_primary_key/a~2Fb,~2Ec-d">a/b,.c-d</a></td>', '<td class="col-pk1 type-str">a/b</td>', '<td class="col-pk2 type-str">.c-d</td>', '<td class="col-content type-str">c</td>', diff --git a/tests/test_utils.py b/tests/test_utils.py index 1c3ab495..790aadc7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,8 +19,8 @@ from unittest.mock import patch ("foo", ["foo"]), ("foo,bar", ["foo", "bar"]), ("123,433,112", ["123", "433", "112"]), - ("123%2C433,112", ["123,433", "112"]), - ("123%2F433%2F112", ["123/433/112"]), + ("123~2C433,112", ["123,433", "112"]), + ("123~2F433~2F112", ["123/433/112"]), ], ) def test_urlsafe_components(path, expected): @@ -93,7 +93,7 @@ def test_path_with_replaced_args(path, args, expected): "row,pks,expected_path", [ ({"A": "foo", "B": "bar"}, ["A", "B"], "foo,bar"), - ({"A": "f,o", "B": "bar"}, ["A", "B"], "f-2Co,bar"), + ({"A": "f,o", "B": "bar"}, ["A", "B"], "f~2Co,bar"), ({"A": 123}, ["A"], "123"), ( utils.CustomRow( @@ -393,9 +393,7 @@ def test_table_columns(): ("/foo?sql=select+1", "json", {}, "/foo.json?sql=select+1"), ("/foo/bar", "json", {}, "/foo/bar.json"), ("/foo/bar", "csv", {}, "/foo/bar.csv"), - ("/foo/bar.csv", "json", {}, "/foo/bar.csv?_format=json"), ("/foo/bar", "csv", {"_dl": 1}, "/foo/bar.csv?_dl=1"), - ("/foo/b.csv", "json", {"_dl": 1}, "/foo/b.csv?_dl=1&_format=json"), ( "/sf-trees/Street_Tree_List?_search=cherry&_size=1000", "csv", @@ -410,18 +408,6 @@ def test_path_with_format(path, format, extra_qs, expected): assert expected == actual -def test_path_with_format_replace_format(): - request = Request.fake("/foo/bar.csv") - assert ( - utils.path_with_format(request=request, format="blob") - == "/foo/bar.csv?_format=blob" - ) - assert ( - utils.path_with_format(request=request, format="blob", replace_format="csv") - == "/foo/bar.blob" - ) - - @pytest.mark.parametrize( "bytes,expected", [ @@ -652,15 +638,15 @@ async def test_derive_named_parameters(sql, expected): "original,expected", ( ("abc", "abc"), - ("/foo/bar", "-2Ffoo-2Fbar"), - ("/-/bar", "-2F-2D-2Fbar"), - ("-/db-/table.csv", "-2D-2Fdb-2D-2Ftable-2Ecsv"), - (r"%~-/", "-25-7E-2D-2F"), - ("-25-7E-2D-2F", "-2D25-2D7E-2D2D-2D2F"), + ("/foo/bar", "~2Ffoo~2Fbar"), + ("/-/bar", "~2F-~2Fbar"), + ("-/db-/table.csv", "-~2Fdb-~2Ftable~2Ecsv"), + (r"%~-/", "~25~7E-~2F"), + ("~25~7E~2D~2F", "~7E25~7E7E~7E2D~7E2F"), ), ) -def test_dash_encoding(original, expected): - actual = utils.dash_encode(original) +def test_tilde_encoding(original, expected): + actual = utils.tilde_encode(original) assert actual == expected # And test round-trip - assert original == utils.dash_decode(actual) + assert original == utils.tilde_decode(actual) From 77a904fea14f743560af9cc668146339bdbbd0a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Mar 2022 11:03:01 -0700 Subject: [PATCH 0641/1866] Update pytest requirement from <7.1.0,>=5.2.2 to >=5.2.2,<7.2.0 (#1656) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...7.1.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e70839d6..4b58b8c4 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "test": [ - "pytest>=5.2.2,<7.1.0", + "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.17,<0.19", "beautifulsoup4>=4.8.1,<4.11.0", From 30e5f0e67c38054a8087a2a4eae3fc4d1779af90 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 17 Mar 2022 14:30:02 -0700 Subject: [PATCH 0642/1866] Documented internals used by datasette-hashed-urls Closes #1663 --- docs/internals.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 3d223603..117cb95c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -217,12 +217,18 @@ You can create your own instance of this - for example to help write tests for a } }) +Constructor parameters include: + +- ``files=[...]`` - a list of database files to open +- ``immutables=[...]`` - a list of database files to open in immutable mode +- ``metadata={...}`` - a dictionary of :ref:`metadata` + .. _datasette_databases: .databases ---------- -Property exposing an ordered dictionary of databases currently connected to Datasette. +Property exposing a ``collections.OrderedDict`` of databases currently connected to Datasette. The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances. @@ -582,6 +588,13 @@ The arguments are as follows: The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. +.. _database_hash: + +db.hash +------- + +If the database was opened in immutable mode, this property returns the 64 character SHA-256 hash of the database contents as a string. Otherwise it returns ``None``. + .. _database_execute: await db.execute(sql, ...) From d4f60c2388c01ddce1b16f95c16d310e037c9912 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 17:12:03 -0700 Subject: [PATCH 0643/1866] Remove hashed URL mode Also simplified how view class routing works. Refs #1661 --- datasette/app.py | 2 +- datasette/views/base.py | 153 ++++++----------------------------- datasette/views/database.py | 19 +++-- datasette/views/index.py | 3 +- datasette/views/special.py | 3 +- datasette/views/table.py | 34 ++++---- tests/fixtures.py | 6 -- tests/test_api.py | 29 ------- tests/test_custom_pages.py | 42 +++++----- tests/test_html.py | 28 ------- tests/test_internals_urls.py | 18 ----- tests/test_table_api.py | 8 -- 12 files changed, 79 insertions(+), 266 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b39ef7cd..3099ada7 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1097,7 +1097,7 @@ class Datasette: ) add_route( TableView.as_view(self), - r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", + r"/(?P<db_name>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$", ) add_route( RowView.as_view(self), diff --git a/datasette/views/base.py b/datasette/views/base.py index 1c0c3f9b..e31beb19 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -122,11 +122,11 @@ class BaseView: async def delete(self, request, *args, **kwargs): return Response.text("Method not allowed", status=405) - async def dispatch_request(self, request, *args, **kwargs): + async def dispatch_request(self, request): if self.ds: await self.ds.refresh_schemas() handler = getattr(self, request.method.lower(), None) - return await handler(request, *args, **kwargs) + return await handler(request) async def render(self, templates, request, context=None): context = context or {} @@ -169,9 +169,7 @@ class BaseView: def as_view(cls, *class_args, **class_kwargs): async def view(request, send): self = view.view_class(*class_args, **class_kwargs) - return await self.dispatch_request( - request, **request.scope["url_route"]["kwargs"] - ) + return await self.dispatch_request(request) view.view_class = cls view.__doc__ = cls.__doc__ @@ -200,90 +198,14 @@ class DataView(BaseView): add_cors_headers(r.headers) return r - async def data(self, request, database, hash, **kwargs): + async def data(self, request): raise NotImplementedError - async def resolve_db_name(self, request, db_name, **kwargs): - hash = None - name = None - decoded_name = tilde_decode(db_name) - if decoded_name not in self.ds.databases and "-" in db_name: - # No matching DB found, maybe it's a name-hash? - name_bit, hash_bit = db_name.rsplit("-", 1) - if tilde_decode(name_bit) not in self.ds.databases: - raise NotFound(f"Database not found: {name}") - else: - name = tilde_decode(name_bit) - hash = hash_bit - else: - name = decoded_name - - try: - db = self.ds.databases[name] - except KeyError: - raise NotFound(f"Database not found: {name}") - - # Verify the hash - expected = "000" - if db.hash is not None: - expected = db.hash[:HASH_LENGTH] - correct_hash_provided = expected == hash - - if not correct_hash_provided: - if "table_and_format" in kwargs: - - async def async_table_exists(t): - return await db.table_exists(t) - - table, _format = await resolve_table_and_format( - table_and_format=tilde_decode(kwargs["table_and_format"]), - table_exists=async_table_exists, - allowed_formats=self.ds.renderers.keys(), - ) - kwargs["table"] = table - if _format: - kwargs["as_format"] = f".{_format}" - elif kwargs.get("table"): - kwargs["table"] = tilde_decode(kwargs["table"]) - - should_redirect = self.ds.urls.path(f"{name}-{expected}") - if kwargs.get("table"): - should_redirect += "/" + tilde_encode(kwargs["table"]) - if kwargs.get("pk_path"): - should_redirect += "/" + kwargs["pk_path"] - if kwargs.get("as_format"): - should_redirect += kwargs["as_format"] - if kwargs.get("as_db"): - should_redirect += kwargs["as_db"] - - if ( - (self.ds.setting("hash_urls") or "_hash" in request.args) - and - # Redirect only if database is immutable - not self.ds.databases[name].is_mutable - ): - return name, expected, correct_hash_provided, should_redirect - - return name, expected, correct_hash_provided, None - def get_templates(self, database, table=None): assert NotImplemented - async def get(self, request, db_name, **kwargs): - ( - database, - hash, - correct_hash_provided, - should_redirect, - ) = await self.resolve_db_name(request, db_name, **kwargs) - if should_redirect: - return self.redirect(request, should_redirect, remove_args={"_hash"}) - - return await self.view_get( - request, database, hash, correct_hash_provided, **kwargs - ) - - async def as_csv(self, request, database, hash, **kwargs): + async def as_csv(self, request, database): + kwargs = {} stream = request.args.get("_stream") # Do not calculate facets or counts: extra_parameters = [ @@ -313,9 +235,7 @@ class DataView(BaseView): kwargs["_size"] = "max" # Fetch the first page try: - response_or_template_contexts = await self.data( - request, database, hash, **kwargs - ) + response_or_template_contexts = await self.data(request) if isinstance(response_or_template_contexts, Response): return response_or_template_contexts elif len(response_or_template_contexts) == 4: @@ -367,10 +287,11 @@ class DataView(BaseView): next = None while first or (next and stream): try: + kwargs = {} if next: kwargs["_next"] = next if not first: - data, _, _ = await self.data(request, database, hash, **kwargs) + data, _, _ = await self.data(request, **kwargs) if first: if request.args.get("_header") != "off": await writer.writerow(headings) @@ -445,60 +366,39 @@ class DataView(BaseView): if not trace: content_type = "text/csv; charset=utf-8" disposition = 'attachment; filename="{}.csv"'.format( - kwargs.get("table", database) + request.url_vars.get("table", database) ) headers["content-disposition"] = disposition return AsgiStream(stream_fn, headers=headers, content_type=content_type) - async def get_format(self, request, database, args): - """Determine the format of the response from the request, from URL - parameters or from a file extension. - - `args` is a dict of the path components parsed from the URL by the router. - """ - # If ?_format= is provided, use that as the format - _format = request.args.get("_format", None) - if not _format: - _format = (args.pop("as_format", None) or "").lstrip(".") + def get_format(self, request): + # Format is the bit from the path following the ., if one exists + last_path_component = request.path.split("/")[-1] + if "." in last_path_component: + return last_path_component.split(".")[-1] else: - args.pop("as_format", None) - if "table_and_format" in args: - db = self.ds.databases[database] + return None - async def async_table_exists(t): - return await db.table_exists(t) - - table, _ext_format = await resolve_table_and_format( - table_and_format=tilde_decode(args["table_and_format"]), - table_exists=async_table_exists, - allowed_formats=self.ds.renderers.keys(), - ) - _format = _format or _ext_format - args["table"] = table - del args["table_and_format"] - elif "table" in args: - args["table"] = tilde_decode(args["table"]) - return _format, args - - async def view_get(self, request, database, hash, correct_hash_provided, **kwargs): - _format, kwargs = await self.get_format(request, database, kwargs) + async def get(self, request): + db_name = request.url_vars["db_name"] + database = tilde_decode(db_name) + _format = self.get_format(request) + data_kwargs = {} if _format == "csv": - return await self.as_csv(request, database, hash, **kwargs) + return await self.as_csv(request, database) if _format is None: # HTML views default to expanding all foreign key labels - kwargs["default_labels"] = True + data_kwargs["default_labels"] = True extra_template_data = {} start = time.perf_counter() status_code = None templates = [] try: - response_or_template_contexts = await self.data( - request, database, hash, **kwargs - ) + response_or_template_contexts = await self.data(request, **data_kwargs) if isinstance(response_or_template_contexts, Response): return response_or_template_contexts # If it has four items, it includes an HTTP status code @@ -650,10 +550,7 @@ class DataView(BaseView): ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): - if correct_hash_provided: - ttl = self.ds.setting("default_cache_ttl_hashed") - else: - ttl = self.ds.setting("default_cache_ttl") + ttl = self.ds.setting("default_cache_ttl") return self.set_response_headers(r, ttl) diff --git a/datasette/views/database.py b/datasette/views/database.py index e26706e7..48635e01 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,6 +12,7 @@ from datasette.utils import ( await_me_maybe, check_visibility, derive_named_parameters, + tilde_decode, to_css_class, validate_sql_select, is_url, @@ -21,7 +22,7 @@ from datasette.utils import ( sqlite3, InvalidSql, ) -from datasette.utils.asgi import AsgiFileDownload, Response, Forbidden +from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm from .base import DatasetteError, DataView @@ -30,7 +31,8 @@ from .base import DatasetteError, DataView class DatabaseView(DataView): name = "database" - async def data(self, request, database, hash, default_labels=False, _size=None): + async def data(self, request, default_labels=False, _size=None): + database = tilde_decode(request.url_vars["db_name"]) await self.check_permissions( request, [ @@ -45,10 +47,13 @@ class DatabaseView(DataView): sql = request.args.get("sql") validate_sql_select(sql) return await QueryView(self.ds).data( - request, database, hash, sql, _size=_size, metadata=metadata + request, sql, _size=_size, metadata=metadata ) - db = self.ds.databases[database] + try: + db = self.ds.databases[database] + except KeyError: + raise NotFound("Database not found: {}".format(database)) table_counts = await db.table_counts(5) hidden_table_names = set(await db.hidden_table_names()) @@ -156,7 +161,8 @@ class DatabaseView(DataView): class DatabaseDownload(DataView): name = "database_download" - async def view_get(self, request, database, hash, correct_hash_present, **kwargs): + async def get(self, request): + database = tilde_decode(request.url_vars["db_name"]) await self.check_permissions( request, [ @@ -191,8 +197,6 @@ class QueryView(DataView): async def data( self, request, - database, - hash, sql, editable=True, canned_query=None, @@ -201,6 +205,7 @@ class QueryView(DataView): named_parameters=None, write=False, ): + database = tilde_decode(request.url_vars["db_name"]) params = {key: request.args.get(key) for key in request.args} if "sql" in params: params.pop("sql") diff --git a/datasette/views/index.py b/datasette/views/index.py index 18454759..311a49db 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -18,7 +18,8 @@ COUNT_DB_SIZE_LIMIT = 100 * 1024 * 1024 class IndexView(BaseView): name = "index" - async def get(self, request, as_format): + async def get(self, request): + as_format = request.url_vars["as_format"] await self.check_permission(request, "view-instance") databases = [] for name, db in self.ds.databases.items(): diff --git a/datasette/views/special.py b/datasette/views/special.py index cdd530f0..c7b5061f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -14,7 +14,8 @@ class JsonDataView(BaseView): self.data_callback = data_callback self.needs_request = needs_request - async def get(self, request, as_format): + async def get(self, request): + as_format = request.url_vars["as_format"] await self.check_permission(request, "view-instance") if self.needs_request: data = self.data_callback(request) diff --git a/datasette/views/table.py b/datasette/views/table.py index 72b8e9a4..8bdc7417 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -271,20 +271,18 @@ class RowTableShared(DataView): class TableView(RowTableShared): name = "table" - async def post(self, request, db_name, table_and_format): + async def post(self, request): + db_name = tilde_decode(request.url_vars["db_name"]) + table = tilde_decode(request.url_vars["table"]) # Handle POST to a canned query - canned_query = await self.ds.get_canned_query( - db_name, table_and_format, request.actor - ) + canned_query = await self.ds.get_canned_query(db_name, table, request.actor) assert canned_query, "You may only POST to a canned query" return await QueryView(self.ds).data( request, - db_name, - None, canned_query["sql"], metadata=canned_query, editable=False, - canned_query=table_and_format, + canned_query=table, named_parameters=canned_query.get("params"), write=bool(canned_query.get("write")), ) @@ -325,20 +323,22 @@ class TableView(RowTableShared): async def data( self, request, - database, - hash, - table, default_labels=False, _next=None, _size=None, ): + database = tilde_decode(request.url_vars["db_name"]) + table = tilde_decode(request.url_vars["table"]) + try: + db = self.ds.databases[database] + except KeyError: + raise NotFound("Database not found: {}".format(database)) + # If this is a canned query, not a table, then dispatch to QueryView instead canned_query = await self.ds.get_canned_query(database, table, request.actor) if canned_query: return await QueryView(self.ds).data( request, - database, - hash, canned_query["sql"], metadata=canned_query, editable=False, @@ -347,9 +347,6 @@ class TableView(RowTableShared): write=bool(canned_query.get("write")), ) - table = tilde_decode(table) - - db = self.ds.databases[database] is_view = bool(await db.get_view_definition(table)) table_exists = bool(await db.table_exists(table)) @@ -940,8 +937,9 @@ async def _sql_params_pks(db, table, pk_values): class RowView(RowTableShared): name = "row" - async def data(self, request, database, hash, table, pk_path, default_labels=False): - table = tilde_decode(table) + async def data(self, request, default_labels=False): + database = tilde_decode(request.url_vars["db_name"]) + table = tilde_decode(request.url_vars["table"]) await self.check_permissions( request, [ @@ -950,7 +948,7 @@ class RowView(RowTableShared): "view-instance", ], ) - pk_values = urlsafe_components(pk_path) + pk_values = urlsafe_components(request.url_vars["pk_path"]) db = self.ds.databases[database] sql, params, pks = await _sql_params_pks(db, table, pk_values) results = await db.execute(sql, params, truncate=True) diff --git a/tests/fixtures.py b/tests/fixtures.py index 11f09c41..342a3020 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -214,12 +214,6 @@ def app_client_two_attached_databases_one_immutable(): yield client -@pytest.fixture(scope="session") -def app_client_with_hash(): - with make_app_client(settings={"hash_urls": True}, is_immutable=True) as client: - yield client - - @pytest.fixture(scope="session") def app_client_with_trace(): with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client: diff --git a/tests/test_api.py b/tests/test_api.py index 87d91e56..46e41afb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -825,35 +825,6 @@ def test_config_redirects_to_settings(app_client, path, expected_redirect): assert response.headers["Location"] == expected_redirect -@pytest.mark.parametrize( - "path,expected_redirect", - [ - ("/fixtures/facetable.json?_hash=1", "/fixtures-HASH/facetable.json"), - ( - "/fixtures/facetable.json?city_id=1&_hash=1", - "/fixtures-HASH/facetable.json?city_id=1", - ), - ], -) -def test_hash_parameter( - app_client_two_attached_databases_one_immutable, path, expected_redirect -): - # First get the current hash for the fixtures database - current_hash = app_client_two_attached_databases_one_immutable.ds.databases[ - "fixtures" - ].hash[:7] - response = app_client_two_attached_databases_one_immutable.get(path) - assert response.status == 302 - location = response.headers["Location"] - assert expected_redirect.replace("HASH", current_hash) == location - - -def test_hash_parameter_ignored_for_mutable_databases(app_client): - path = "/fixtures/facetable.json?_hash=1" - response = app_client.get(path) - assert response.status == 200 - - test_json_columns_default_expected = [ {"intval": 1, "strval": "s", "floatval": 0.5, "jsonval": '{"foo": "bar"}'} ] diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 66b7437a..f2cfe394 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -21,61 +21,61 @@ def custom_pages_client_with_base_url(): def test_custom_pages_view_name(custom_pages_client): response = custom_pages_client.get("/about") - assert 200 == response.status - assert "ABOUT! view_name:page" == response.text + assert response.status == 200 + assert response.text == "ABOUT! view_name:page" def test_request_is_available(custom_pages_client): response = custom_pages_client.get("/request") - assert 200 == response.status - assert "path:/request" == response.text + assert response.status == 200 + assert response.text == "path:/request" def test_custom_pages_with_base_url(custom_pages_client_with_base_url): response = custom_pages_client_with_base_url.get("/prefix/request") - assert 200 == response.status - assert "path:/prefix/request" == response.text + assert response.status == 200 + assert response.text == "path:/prefix/request" def test_custom_pages_nested(custom_pages_client): response = custom_pages_client.get("/nested/nest") - assert 200 == response.status - assert "Nest!" == response.text + assert response.status == 200 + assert response.text == "Nest!" response = custom_pages_client.get("/nested/nest2") - assert 404 == response.status + assert response.status == 404 def test_custom_status(custom_pages_client): response = custom_pages_client.get("/202") - assert 202 == response.status - assert "202!" == response.text + assert response.status == 202 + assert response.text == "202!" def test_custom_headers(custom_pages_client): response = custom_pages_client.get("/headers") - assert 200 == response.status - assert "foo" == response.headers["x-this-is-foo"] - assert "bar" == response.headers["x-this-is-bar"] - assert "FOOBAR" == response.text + assert response.status == 200 + assert response.headers["x-this-is-foo"] == "foo" + assert response.headers["x-this-is-bar"] == "bar" + assert response.text == "FOOBAR" def test_custom_content_type(custom_pages_client): response = custom_pages_client.get("/atom") - assert 200 == response.status + assert response.status == 200 assert response.headers["content-type"] == "application/xml" - assert "<?xml ...>" == response.text + assert response.text == "<?xml ...>" def test_redirect(custom_pages_client): response = custom_pages_client.get("/redirect") - assert 302 == response.status - assert "/example" == response.headers["Location"] + assert response.status == 302 + assert response.headers["Location"] == "/example" def test_redirect2(custom_pages_client): response = custom_pages_client.get("/redirect2") - assert 301 == response.status - assert "/example" == response.headers["Location"] + assert response.status == 301 + assert response.headers["Location"] == "/example" @pytest.mark.parametrize( diff --git a/tests/test_html.py b/tests/test_html.py index 76a8423a..6e4c22b1 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -5,7 +5,6 @@ from .fixtures import ( # noqa app_client_base_url_prefix, app_client_shorter_time_limit, app_client_two_attached_databases, - app_client_with_hash, make_app_client, METADATA, ) @@ -101,13 +100,6 @@ def test_not_allowed_methods(): assert response.status == 405 -def test_database_page_redirects_with_url_hash(app_client_with_hash): - response = app_client_with_hash.get("/fixtures") - assert response.status == 302 - response = app_client_with_hash.get("/fixtures", follow_redirects=True) - assert "fixtures" in response.text - - def test_database_page(app_client): response = app_client.get("/fixtures") soup = Soup(response.body, "html.parser") @@ -182,26 +174,6 @@ def test_sql_time_limit(app_client_shorter_time_limit): assert expected_html_fragment in response.text -def test_row_redirects_with_url_hash(app_client_with_hash): - response = app_client_with_hash.get("/fixtures/simple_primary_key/1") - assert response.status == 302 - assert response.headers["Location"].endswith("/1") - response = app_client_with_hash.get( - "/fixtures/simple_primary_key/1", follow_redirects=True - ) - assert response.status == 200 - - -def test_row_strange_table_name_with_url_hash(app_client_with_hash): - response = app_client_with_hash.get("/fixtures/table~2Fwith~2Fslashes~2Ecsv/3") - assert response.status == 302 - assert response.headers["Location"].endswith("/table~2Fwith~2Fslashes~2Ecsv/3") - response = app_client_with_hash.get( - "/fixtures/table~2Fwith~2Fslashes~2Ecsv/3", follow_redirects=True - ) - assert response.status == 200 - - def test_row_page_does_not_truncate(): with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable/1") diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 4307789c..d60aafcf 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,6 +1,5 @@ from datasette.app import Datasette from datasette.utils import PrefixedUrlString -from .fixtures import app_client_with_hash import pytest @@ -147,20 +146,3 @@ def test_row(ds, base_url, format, expected): actual = ds.urls.row("_memory", "facetable", "1", format=format) assert actual == expected assert isinstance(actual, PrefixedUrlString) - - -@pytest.mark.parametrize("base_url", ["/", "/prefix/"]) -def test_database_hashed(app_client_with_hash, base_url): - ds = app_client_with_hash.ds - original_base_url = ds._settings["base_url"] - try: - ds._settings["base_url"] = base_url - db_hash = ds.get_database("fixtures").hash - assert len(db_hash) == 64 - expected = f"{base_url}fixtures-{db_hash[:7]}" - assert ds.urls.database("fixtures") == expected - assert ds.urls.table("fixtures", "name") == expected + "/name" - assert ds.urls.query("fixtures", "name") == expected + "/name" - finally: - # Reset this since fixture is shared with other tests - ds._settings["base_url"] = original_base_url diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 3ab369b3..3d0a7fbd 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -2,7 +2,6 @@ from datasette.utils import detect_json1 from datasette.utils.sqlite import sqlite_version from .fixtures import ( # noqa app_client, - app_client_with_hash, app_client_with_trace, app_client_returned_rows_matches_page_size, generate_compound_rows, @@ -41,13 +40,6 @@ def test_table_not_exists_json(app_client): } == app_client.get("/fixtures/blah.json").json -def test_jsono_redirects_to_shape_objects(app_client_with_hash): - response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono") - response = app_client_with_hash.get(response_1.headers["Location"]) - assert response.status == 302 - assert response.headers["Location"].endswith("?_shape=objects") - - def test_table_shape_arrays(app_client): response = app_client.get("/fixtures/simple_primary_key.json?_shape=arrays") assert [ From 8658c66438ec71edc7e9adc495f4692b937a0f57 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 17:19:31 -0700 Subject: [PATCH 0644/1866] Show error if --setting hash_urls 1 used, refs #1661 --- datasette/app.py | 15 ++++++++++----- datasette/cli.py | 21 ++++++++++++++++++--- tests/test_cli.py | 7 +++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3099ada7..c1c0663d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -118,11 +118,6 @@ SETTINGS = ( 50, "Time limit for calculating a suggested facet", ), - Setting( - "hash_urls", - False, - "Include DB file contents hash in URLs, for far-future caching", - ), Setting( "allow_facet", True, @@ -177,6 +172,16 @@ SETTINGS = ( ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) +OBSOLETE_SETTINGS = { + option.name: option + for option in ( + Setting( + "hash_urls", + False, + "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead", + ), + ) +} DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} diff --git a/datasette/cli.py b/datasette/cli.py index 61e7ce91..b94ac192 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -12,7 +12,14 @@ from subprocess import call import sys from runpy import run_module import webbrowser -from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm +from .app import ( + OBSOLETE_SETTINGS, + Datasette, + DEFAULT_SETTINGS, + SETTINGS, + SQLITE_LIMIT_ATTACHED, + pm, +) from .utils import ( StartupError, check_connection, @@ -50,8 +57,12 @@ class Config(click.ParamType): return name, value = config.split(":", 1) if name not in DEFAULT_SETTINGS: + if name in OBSOLETE_SETTINGS: + msg = OBSOLETE_SETTINGS[name].help + else: + msg = f"{name} is not a valid option (--help-settings to see all)" self.fail( - f"{name} is not a valid option (--help-settings to see all)", + msg, param, ctx, ) @@ -83,8 +94,12 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config if name not in DEFAULT_SETTINGS: + if name in OBSOLETE_SETTINGS: + msg = OBSOLETE_SETTINGS[name].help + else: + msg = f"{name} is not a valid option (--help-settings to see all)" self.fail( - f"{name} is not a valid option (--help-settings to see all)", + msg, param, ctx, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5afe72c1..89e8d044 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -310,3 +310,10 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output + + +def test_help_error_on_hash_urls_setting(): + runner = CliRunner() + result = runner.invoke(cli, ["--setting", "hash_urls", 1]) + assert result.exit_code == 2 + assert 'The hash_urls setting has been removed' in result.output From 9979dcd07f9921ac30c4c0b5ea60d09cd1e10556 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 17:25:14 -0700 Subject: [PATCH 0645/1866] Also remove default_cache_ttl_hashed setting, refs #1661 --- datasette/app.py | 17 +++-------------- datasette/cli.py | 16 ++++++++-------- datasette/url_builder.py | 9 +-------- tests/test_api.py | 2 -- tests/test_cli.py | 7 ++++--- 5 files changed, 16 insertions(+), 35 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c1c0663d..f52e3283 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -134,11 +134,6 @@ SETTINGS = ( 5, "Default HTTP cache TTL (used in Cache-Control: max-age= header)", ), - Setting( - "default_cache_ttl_hashed", - 365 * 24 * 60 * 60, - "Default HTTP cache TTL for hashed URL pages", - ), Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"), Setting( "allow_csv_stream", @@ -172,17 +167,11 @@ SETTINGS = ( ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) +_HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead" OBSOLETE_SETTINGS = { - option.name: option - for option in ( - Setting( - "hash_urls", - False, - "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead", - ), - ) + "hash_urls": _HASH_URLS_REMOVED, + "default_cache_ttl_hashed": _HASH_URLS_REMOVED, } - DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" diff --git a/datasette/cli.py b/datasette/cli.py index b94ac192..3c6e1b2c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -57,10 +57,10 @@ class Config(click.ParamType): return name, value = config.split(":", 1) if name not in DEFAULT_SETTINGS: - if name in OBSOLETE_SETTINGS: - msg = OBSOLETE_SETTINGS[name].help - else: - msg = f"{name} is not a valid option (--help-settings to see all)" + msg = ( + OBSOLETE_SETTINGS.get(name) + or f"{name} is not a valid option (--help-settings to see all)" + ) self.fail( msg, param, @@ -94,10 +94,10 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config if name not in DEFAULT_SETTINGS: - if name in OBSOLETE_SETTINGS: - msg = OBSOLETE_SETTINGS[name].help - else: - msg = f"{name} is not a valid option (--help-settings to see all)" + msg = ( + OBSOLETE_SETTINGS.get(name) + or f"{name} is not a valid option (--help-settings to see all)" + ) self.fail( msg, param, diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 9f072462..498ec85d 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -28,14 +28,7 @@ class Urls: return self.path("-/logout") def database(self, database, format=None): - db = self.ds.databases[database] - if self.ds.setting("hash_urls") and db.hash: - path = self.path( - f"{tilde_encode(database)}-{db.hash[:HASH_LENGTH]}", format=format - ) - else: - path = self.path(tilde_encode(database), format=format) - return path + return self.path(tilde_encode(database), format=format) def table(self, database, table, format=None): path = f"{self.database(database)}/{tilde_encode(table)}" diff --git a/tests/test_api.py b/tests/test_api.py index 46e41afb..d3c94023 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -798,14 +798,12 @@ def test_settings_json(app_client): "allow_facet": True, "suggest_facets": True, "default_cache_ttl": 5, - "default_cache_ttl_hashed": 365 * 24 * 60 * 60, "num_sql_threads": 1, "cache_size_kb": 0, "allow_csv_stream": True, "max_csv_mb": 100, "truncate_cells_html": 2048, "force_https_urls": False, - "hash_urls": False, "template_debug": False, "trace_debug": False, "base_url": "/", diff --git a/tests/test_cli.py b/tests/test_cli.py index 89e8d044..dca65f26 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -312,8 +312,9 @@ def test_help_settings(): assert setting.name in result.output -def test_help_error_on_hash_urls_setting(): +@pytest.mark.parametrize("setting", ("hash_urls", "default_cache_ttl_hashed")) +def test_help_error_on_hash_urls_setting(setting): runner = CliRunner() - result = runner.invoke(cli, ["--setting", "hash_urls", 1]) + result = runner.invoke(cli, ["--setting", setting, 1]) assert result.exit_code == 2 - assert 'The hash_urls setting has been removed' in result.output + assert "The hash_urls setting has been removed" in result.output From 32963018e7edfab1233de7c7076c428d0e5c7813 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 17:33:06 -0700 Subject: [PATCH 0646/1866] Updated documentation to remove hash_urls, refs #1661 --- docs/cli-reference.rst | 4 ---- docs/performance.rst | 18 +++++++++++------- docs/settings.rst | 27 --------------------------- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 155a005d..69670d8a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -142,8 +142,6 @@ datasette serve --help-settings (default=200) facet_suggest_time_limit_ms Time limit for calculating a suggested facet (default=50) - hash_urls Include DB file contents hash in URLs, for far- - future caching (default=False) allow_facet Allow users to specify columns to facet using ?_facet= parameter (default=True) allow_download Allow users to download the original SQLite @@ -152,8 +150,6 @@ datasette serve --help-settings (default=True) default_cache_ttl Default HTTP cache TTL (used in Cache-Control: max-age= header) (default=5) - default_cache_ttl_hashed Default HTTP cache TTL for hashed URL pages - (default=31536000) cache_size_kb SQLite cache size in KB (0 == use SQLite default) (default=0) allow_csv_stream Allow .csv?_stream=1 to download all rows diff --git a/docs/performance.rst b/docs/performance.rst index bcf3208e..d37f1804 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -60,18 +60,22 @@ The :ref:`setting_default_cache_ttl` setting sets the default HTTP cache TTL for You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` query string parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query. -Hashed URL mode ---------------- +datasette-hashed-urls +--------------------- -When you open a database file in immutable mode using the ``-i`` option, Datasette calculates a SHA-256 hash of the contents of that file on startup. This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future. This results in URLs that can then be cached indefinitely by both browsers and caching proxies - an enormous potential performance optimization. +If you open a database file in immutable mode using the ``-i`` option, you can be assured that the content of that database will not change for the lifetime of the Datasette server. -You can enable these hashed URLs in two ways: using the :ref:`setting_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` query string parameter (which only applies to the current request). +The `datasette-hashed-urls plugin <https://datasette.io/plugins/datasette-hashed-urls>`__ implements an optimization where your database is served with part of the SHA-256 hash of the database contents baked into the URL. -With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`setting_default_cache_ttl_hashed` which defaults to 365 days. +A database at ``/fixtures`` will instead be served at ``/fixtures-aa7318b``, and a year-long cache expiry header will be returned with those pages. -Since these responses are cached for a long time, you may wish to build API clients against the non-hashed version of these URLs. These 302 redirects are served extremely quickly, so this should still be a performant way to work against the Datasette API. +This will then be cached by both browsers and caching proxies such as Cloudflare or Fastly, providing a potentially significant performance boost. -If you run Datasette behind an `HTTP/2 server push <https://en.wikipedia.org/wiki/HTTP/2_Server_Push>`__ aware proxy such as Cloudflare Datasette will serve the 302 redirects in such a way that the redirected page will be efficiently "pushed" to the browser as part of the response, without the browser needing to make a second HTTP request to fetch the redirected resource. +To install the plugin, run the following:: + + datasette install datasette-hashed-urls .. note:: + Prior to Datasette 0.61 hashed URL mode was a core Datasette feature, enabled using the ``hash_urls`` setting. This implementation has now been removed in favor of the ``datasette-hashed-urls`` plugin. + Prior to Datasette 0.28 hashed URL mode was the default behaviour for Datasette, since all database files were assumed to be immutable and unchanging. From 0.28 onwards the default has been to treat database files as mutable unless explicitly configured otherwise. diff --git a/docs/settings.rst b/docs/settings.rst index da06d6a0..60c4b36d 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -178,17 +178,6 @@ Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-ag datasette mydatabase.db --setting default_cache_ttl 60 -.. _setting_default_cache_ttl_hashed: - -default_cache_ttl_hashed -~~~~~~~~~~~~~~~~~~~~~~~~ - -Default HTTP caching max-age for responses served using using the :ref:`hashed-urls mechanism <setting_hash_urls>`. Defaults to 365 days (31536000 seconds). - -:: - - datasette mydatabase.db --setting default_cache_ttl_hashed 10000 - .. _setting_cache_size_kb: cache_size_kb @@ -251,22 +240,6 @@ HTTP but is served to the outside world via a proxy that enables HTTPS. datasette mydatabase.db --setting force_https_urls 1 -.. _setting_hash_urls: - -hash_urls -~~~~~~~~~ - -When enabled, this setting causes Datasette to append a content hash of the -database file to the URL path for every table and query within that database. - -When combined with far-future expire headers this ensures that queries can be -cached forever, safe in the knowledge that any modifications to the database -itself will result in new, uncached URL paths. - -:: - - datasette mydatabase.db --setting hash_urls 1 - .. _setting_template_debug: template_debug From 4e47a2d894b96854348343374c8e97c9d7055cf6 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 18:37:54 -0700 Subject: [PATCH 0647/1866] Fixed bug where tables with a column called n caused 500 errors Closes #1228 --- datasette/facets.py | 6 +++--- tests/fixtures.py | 33 ++++++++++++++++---------------- tests/test_api.py | 1 + tests/test_csv.py | 32 +++++++++++++++---------------- tests/test_internals_database.py | 10 ++++++++++ tests/test_plugins.py | 6 ++++-- tests/test_table_api.py | 8 ++++++++ 7 files changed, 59 insertions(+), 37 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index a1bb4a5f..b15a758c 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -151,10 +151,10 @@ class ColumnFacet(Facet): if column in already_enabled: continue suggested_facet_sql = """ - select {column}, count(*) as n from ( + select {column} as value, count(*) as n from ( {sql} - ) where {column} is not null - group by {column} + ) where value is not null + group by value limit {limit} """.format( column=escape_sqlite(column), sql=self.sql, limit=facet_size + 1 diff --git a/tests/fixtures.py b/tests/fixtures.py index 342a3020..e0e4ec7b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -564,26 +564,27 @@ CREATE TABLE facetable ( tags text, complex_array text, distinct_some_null, + n text, FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) ); INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null) + (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null) + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) ; CREATE TABLE binary_data ( diff --git a/tests/test_api.py b/tests/test_api.py index d3c94023..421bb1fe 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -210,6 +210,7 @@ def test_database_page(app_client): "tags", "complex_array", "distinct_some_null", + "n", ], "primary_keys": ["pk"], "count": 15, diff --git a/tests/test_csv.py b/tests/test_csv.py index 8749cd8b..7fc25a09 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -24,22 +24,22 @@ world ) EXPECTED_TABLE_WITH_LABELS_CSV = """ -pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null -1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one -2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two -3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[], -4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[], -5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[], -6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[], -7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[], -8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[], -9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[], -10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[], -11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[], -12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[], -13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[], -14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[], -15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[], +pk,created,planet_int,on_earth,state,_city_id,_city_id_label,_neighborhood,tags,complex_array,distinct_some_null,n +1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one,n1 +2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two,n2 +3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],, +4,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Tenderloin,[],[],, +5,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Bernal Heights,[],[],, +6,2019-01-15 08:00:00,1,1,CA,1,San Francisco,Hayes Valley,[],[],, +7,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Hollywood,[],[],, +8,2019-01-15 08:00:00,1,1,CA,2,Los Angeles,Downtown,[],[],, +9,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Los Feliz,[],[],, +10,2019-01-16 08:00:00,1,1,CA,2,Los Angeles,Koreatown,[],[],, +11,2019-01-16 08:00:00,1,1,MI,3,Detroit,Downtown,[],[],, +12,2019-01-17 08:00:00,1,1,MI,3,Detroit,Greektown,[],[],, +13,2019-01-17 08:00:00,1,1,MI,3,Detroit,Corktown,[],[],, +14,2019-01-17 08:00:00,1,1,MI,3,Detroit,Mexicantown,[],[],, +15,2019-01-17 08:00:00,2,0,MC,4,Memnonia,Arcadia Planitia,[],[],, """.lstrip().replace( "\n", "\r\n" ) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 31538a24..551f67e1 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -86,6 +86,7 @@ async def test_table_exists(db, tables, exists): "tags", "complex_array", "distinct_some_null", + "n", ], ), ( @@ -204,6 +205,15 @@ async def test_table_columns(db, table, expected): is_pk=0, hidden=0, ), + Column( + cid=10, + name="n", + type="text", + notnull=0, + default_value=None, + is_pk=0, + hidden=0, + ), ], ), ( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 656f39e4..15bde962 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -442,6 +442,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): "tags", "complex_array", "distinct_some_null", + "n", ], "rows": [ "<sqlite3.Row object at 0xXXX>", @@ -460,7 +461,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): "<sqlite3.Row object at 0xXXX>", "<sqlite3.Row object at 0xXXX>", ], - "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51", "query_name": None, "database": "fixtures", "table": "facetable", @@ -531,8 +532,9 @@ def test_hook_register_output_renderer_can_render(app_client): "tags", "complex_array", "distinct_some_null", + "n", ], - "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51", + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51", "query_name": None, "database": "fixtures", "table": "facetable", diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 3d0a7fbd..9db383c3 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -532,6 +532,7 @@ def test_table_filter_json_arraycontains(app_client): '["tag1", "tag2"]', '[{"foo": "bar"}]', "one", + "n1", ], [ 2, @@ -544,6 +545,7 @@ def test_table_filter_json_arraycontains(app_client): '["tag1", "tag3"]', "[]", "two", + "n2", ], ] @@ -565,6 +567,7 @@ def test_table_filter_json_arraynotcontains(app_client): '["tag1", "tag2"]', '[{"foo": "bar"}]', "one", + "n1", ] ] @@ -585,6 +588,7 @@ def test_table_filter_extra_where(app_client): '["tag1", "tag3"]', "[]", "two", + "n2", ] ] == response.json["rows"] @@ -958,6 +962,7 @@ def test_expand_labels(app_client): "tags": '["tag1", "tag3"]', "complex_array": "[]", "distinct_some_null": "two", + "n": "n2", }, "13": { "pk": 13, @@ -970,6 +975,7 @@ def test_expand_labels(app_client): "tags": "[]", "complex_array": "[]", "distinct_some_null": None, + "n": None, }, } == response.json @@ -1161,6 +1167,7 @@ def test_generated_columns_are_visible_in_datasette(): "tags", "complex_array", "distinct_some_null", + "n", ], ), ( @@ -1188,6 +1195,7 @@ def test_generated_columns_are_visible_in_datasette(): "tags", "complex_array", "distinct_some_null", + "n", ], ), ( From 711767bcd3c1e76a0861fe7f24069ff1c8efc97a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 18 Mar 2022 21:03:08 -0700 Subject: [PATCH 0648/1866] Refactored URL routing to add tests, closes #1666 Refs #1660 --- datasette/app.py | 54 ++++++++++++++++++++----------------- datasette/utils/__init__.py | 8 ++++++ tests/test_routes.py | 34 +++++++++++++++++++++++ 3 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 tests/test_routes.py diff --git a/datasette/app.py b/datasette/app.py index f52e3283..8987112c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -60,6 +60,7 @@ from .utils import ( module_from_path, parse_metadata, resolve_env_secrets, + resolve_routes, to_css_class, ) from .utils.asgi import ( @@ -974,8 +975,7 @@ class Datasette: output.append(script) return output - def app(self): - """Returns an ASGI app function that serves the whole of Datasette""" + def _routes(self): routes = [] for routes_to_add in pm.hook.register_routes(datasette=self): @@ -1099,6 +1099,15 @@ class Datasette: + renderer_regex + r")?$", ) + return [ + # Compile any strings to regular expressions + ((re.compile(pattern) if isinstance(pattern, str) else pattern), view) + for pattern, view in routes + ] + + def app(self): + """Returns an ASGI app function that serves the whole of Datasette""" + routes = self._routes() self._register_custom_units() async def setup_db(): @@ -1129,12 +1138,7 @@ class Datasette: class DatasetteRouter: def __init__(self, datasette, routes): self.ds = datasette - routes = routes or [] - self.routes = [ - # Compile any strings to regular expressions - ((re.compile(pattern) if isinstance(pattern, str) else pattern), view) - for pattern, view in routes - ] + self.routes = routes or [] # Build a list of pages/blah/{name}.html matching expressions pattern_templates = [ filepath @@ -1187,22 +1191,24 @@ class DatasetteRouter: break scope_modifications["actor"] = actor or default_actor scope = dict(scope, **scope_modifications) - for regex, view in self.routes: - match = regex.match(path) - if match is not None: - new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) - request.scope = new_scope - try: - response = await view(request, send) - if response: - self.ds._write_messages_to_response(request, response) - await response.asgi_send(send) - return - except NotFound as exception: - return await self.handle_404(request, send, exception) - except Exception as exception: - return await self.handle_500(request, send, exception) - return await self.handle_404(request, send) + + match, view = resolve_routes(self.routes, path) + + if match is None: + return await self.handle_404(request, send) + + new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) + request.scope = new_scope + try: + response = await view(request, send) + if response: + self.ds._write_messages_to_response(request, response) + await response.asgi_send(send) + return + except NotFound as exception: + return await self.handle_404(request, send, exception) + except Exception as exception: + return await self.handle_500(request, send, exception) async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bd591459..ccdf8ad4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1178,3 +1178,11 @@ def tilde_decode(s: str) -> str: s = s.replace("%", temp) decoded = urllib.parse.unquote(s.replace("~", "%")) return decoded.replace(temp, "%") + + +def resolve_routes(routes, path): + for regex, view in routes: + match = regex.match(path) + if match is not None: + return match, view + return None, None diff --git a/tests/test_routes.py b/tests/test_routes.py new file mode 100644 index 00000000..a1960f14 --- /dev/null +++ b/tests/test_routes.py @@ -0,0 +1,34 @@ +from datasette.app import Datasette +from datasette.utils import resolve_routes +import pytest + + +@pytest.fixture(scope="session") +def routes(): + ds = Datasette() + return ds._routes() + + +@pytest.mark.parametrize( + "path,expected", + ( + ("/", "IndexView"), + ("/foo", "DatabaseView"), + ("/foo.csv", "DatabaseView"), + ("/foo.json", "DatabaseView"), + ("/foo.humbug", "DatabaseView"), + ("/foo/humbug", "TableView"), + ("/foo/humbug.json", "TableView"), + ("/foo/humbug.blah", "TableView"), + ("/foo/humbug/1", "RowView"), + ("/foo/humbug/1.json", "RowView"), + ("/-/metadata.json", "JsonDataView"), + ("/-/metadata", "JsonDataView"), + ), +) +def test_routes(routes, path, expected): + match, view = resolve_routes(routes, path) + if expected is None: + assert match is None + else: + assert view.view_class.__name__ == expected From 764738dfcb16cd98b0987d443f59d5baa9d3c332 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 09:30:22 -0700 Subject: [PATCH 0649/1866] test_routes also now asserts matches, refs #1666 --- tests/test_routes.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tests/test_routes.py b/tests/test_routes.py index a1960f14..6718c232 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -10,25 +10,34 @@ def routes(): @pytest.mark.parametrize( - "path,expected", + "path,expected_class,expected_matches", ( - ("/", "IndexView"), - ("/foo", "DatabaseView"), - ("/foo.csv", "DatabaseView"), - ("/foo.json", "DatabaseView"), - ("/foo.humbug", "DatabaseView"), - ("/foo/humbug", "TableView"), - ("/foo/humbug.json", "TableView"), - ("/foo/humbug.blah", "TableView"), - ("/foo/humbug/1", "RowView"), - ("/foo/humbug/1.json", "RowView"), - ("/-/metadata.json", "JsonDataView"), - ("/-/metadata", "JsonDataView"), + ("/", "IndexView", {"as_format": ""}), + ("/foo", "DatabaseView", {"as_format": None, "db_name": "foo"}), + ("/foo.csv", "DatabaseView", {"as_format": ".csv", "db_name": "foo"}), + ("/foo.json", "DatabaseView", {"as_format": ".json", "db_name": "foo"}), + ("/foo.humbug", "DatabaseView", {"as_format": None, "db_name": "foo.humbug"}), + ("/foo/humbug", "TableView", {"db_name": "foo", "table": "humbug"}), + ("/foo/humbug.json", "TableView", {"db_name": "foo", "table": "humbug"}), + ("/foo/humbug.blah", "TableView", {"db_name": "foo", "table": "humbug"}), + ( + "/foo/humbug/1", + "RowView", + {"as_format": None, "db_name": "foo", "pk_path": "1", "table": "humbug"}, + ), + ( + "/foo/humbug/1.json", + "RowView", + {"as_format": ".json", "db_name": "foo", "pk_path": "1", "table": "humbug"}, + ), + ("/-/metadata.json", "JsonDataView", {"as_format": ".json"}), + ("/-/metadata", "JsonDataView", {"as_format": ""}), ), ) -def test_routes(routes, path, expected): +def test_routes(routes, path, expected_class, expected_matches): match, view = resolve_routes(routes, path) - if expected is None: + if expected_class is None: assert match is None else: - assert view.view_class.__name__ == expected + assert view.view_class.__name__ == expected_class + assert match.groupdict() == expected_matches From 61419388c134001118aaf7dfb913562d467d7913 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 09:52:08 -0700 Subject: [PATCH 0650/1866] Rename route match groups for consistency, refs #1667, #1660 --- datasette/app.py | 28 ++++++++++++---------------- datasette/blob_renderer.py | 4 ++-- datasette/views/base.py | 2 +- datasette/views/database.py | 6 +++--- datasette/views/index.py | 2 +- datasette/views/special.py | 2 +- datasette/views/table.py | 8 ++++---- tests/test_routes.py | 24 ++++++++++++------------ 8 files changed, 36 insertions(+), 40 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8987112c..5259c50c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -988,7 +988,7 @@ class Datasette: # Generate a regex snippet to match all registered renderer file extensions renderer_regex = "|".join(r"\." + key for key in self.renderers.keys()) - add_route(IndexView.as_view(self), r"/(?P<as_format>(\.jsono?)?$)") + add_route(IndexView.as_view(self), r"/(?P<format>(\.jsono?)?$)") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires add_route(favicon, "/favicon.ico") @@ -1020,21 +1020,21 @@ class Datasette: ) add_route( JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()), - r"/-/metadata(?P<as_format>(\.json)?)$", + r"/-/metadata(?P<format>(\.json)?)$", ) add_route( JsonDataView.as_view(self, "versions.json", self._versions), - r"/-/versions(?P<as_format>(\.json)?)$", + r"/-/versions(?P<format>(\.json)?)$", ) add_route( JsonDataView.as_view( self, "plugins.json", self._plugins, needs_request=True ), - r"/-/plugins(?P<as_format>(\.json)?)$", + r"/-/plugins(?P<format>(\.json)?)$", ) add_route( JsonDataView.as_view(self, "settings.json", lambda: self._settings), - r"/-/settings(?P<as_format>(\.json)?)$", + r"/-/settings(?P<format>(\.json)?)$", ) add_route( permanent_redirect("/-/settings.json"), @@ -1046,15 +1046,15 @@ class Datasette: ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), - r"/-/threads(?P<as_format>(\.json)?)$", + r"/-/threads(?P<format>(\.json)?)$", ) add_route( JsonDataView.as_view(self, "databases.json", self._connected_databases), - r"/-/databases(?P<as_format>(\.json)?)$", + r"/-/databases(?P<format>(\.json)?)$", ) add_route( JsonDataView.as_view(self, "actor.json", self._actor, needs_request=True), - r"/-/actor(?P<as_format>(\.json)?)$", + r"/-/actor(?P<format>(\.json)?)$", ) add_route( AuthTokenView.as_view(self), @@ -1080,22 +1080,18 @@ class Datasette: PatternPortfolioView.as_view(self), r"/-/patterns$", ) - add_route( - DatabaseDownload.as_view(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$" - ) + add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^/]+?)\.db$") add_route( DatabaseView.as_view(self), - r"/(?P<db_name>[^/]+?)(?P<as_format>" - + renderer_regex - + r"|.jsono|\.csv)?$", + r"/(?P<database>[^/]+?)(?P<format>" + renderer_regex + r"|.jsono|\.csv)?$", ) add_route( TableView.as_view(self), - r"/(?P<db_name>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$", + r"/(?P<database>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$", ) add_route( RowView.as_view(self), - r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>" + r"/(?P<database>[^/]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(?P<format>" + renderer_regex + r")?$", ) diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index 217b3638..4d8c6bea 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -34,8 +34,8 @@ async def render_blob(datasette, database, rows, columns, request, table, view_n filename_bits = [] if table: filename_bits.append(to_css_class(table)) - if "pk_path" in request.url_vars: - filename_bits.append(request.url_vars["pk_path"]) + if "pks" in request.url_vars: + filename_bits.append(request.url_vars["pks"]) filename_bits.append(to_css_class(blob_column)) if blob_hash: filename_bits.append(blob_hash[:6]) diff --git a/datasette/views/base.py b/datasette/views/base.py index e31beb19..0bbf98bb 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -381,7 +381,7 @@ class DataView(BaseView): return None async def get(self, request): - db_name = request.url_vars["db_name"] + db_name = request.url_vars["database"] database = tilde_decode(db_name) _format = self.get_format(request) data_kwargs = {} diff --git a/datasette/views/database.py b/datasette/views/database.py index 48635e01..93bd1011 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -32,7 +32,7 @@ class DatabaseView(DataView): name = "database" async def data(self, request, default_labels=False, _size=None): - database = tilde_decode(request.url_vars["db_name"]) + database = tilde_decode(request.url_vars["database"]) await self.check_permissions( request, [ @@ -162,7 +162,7 @@ class DatabaseDownload(DataView): name = "database_download" async def get(self, request): - database = tilde_decode(request.url_vars["db_name"]) + database = tilde_decode(request.url_vars["database"]) await self.check_permissions( request, [ @@ -205,7 +205,7 @@ class QueryView(DataView): named_parameters=None, write=False, ): - database = tilde_decode(request.url_vars["db_name"]) + database = tilde_decode(request.url_vars["database"]) params = {key: request.args.get(key) for key in request.args} if "sql" in params: params.pop("sql") diff --git a/datasette/views/index.py b/datasette/views/index.py index 311a49db..f5e31181 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -19,7 +19,7 @@ class IndexView(BaseView): name = "index" async def get(self, request): - as_format = request.url_vars["as_format"] + as_format = request.url_vars["format"] await self.check_permission(request, "view-instance") databases = [] for name, db in self.ds.databases.items(): diff --git a/datasette/views/special.py b/datasette/views/special.py index c7b5061f..395ee587 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -15,7 +15,7 @@ class JsonDataView(BaseView): self.needs_request = needs_request async def get(self, request): - as_format = request.url_vars["as_format"] + as_format = request.url_vars["format"] await self.check_permission(request, "view-instance") if self.needs_request: data = self.data_callback(request) diff --git a/datasette/views/table.py b/datasette/views/table.py index 8bdc7417..ea4f24b7 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -272,7 +272,7 @@ class TableView(RowTableShared): name = "table" async def post(self, request): - db_name = tilde_decode(request.url_vars["db_name"]) + db_name = tilde_decode(request.url_vars["database"]) table = tilde_decode(request.url_vars["table"]) # Handle POST to a canned query canned_query = await self.ds.get_canned_query(db_name, table, request.actor) @@ -327,7 +327,7 @@ class TableView(RowTableShared): _next=None, _size=None, ): - database = tilde_decode(request.url_vars["db_name"]) + database = tilde_decode(request.url_vars["database"]) table = tilde_decode(request.url_vars["table"]) try: db = self.ds.databases[database] @@ -938,7 +938,7 @@ class RowView(RowTableShared): name = "row" async def data(self, request, default_labels=False): - database = tilde_decode(request.url_vars["db_name"]) + database = tilde_decode(request.url_vars["database"]) table = tilde_decode(request.url_vars["table"]) await self.check_permissions( request, @@ -948,7 +948,7 @@ class RowView(RowTableShared): "view-instance", ], ) - pk_values = urlsafe_components(request.url_vars["pk_path"]) + pk_values = urlsafe_components(request.url_vars["pks"]) db = self.ds.databases[database] sql, params, pks = await _sql_params_pks(db, table, pk_values) results = await db.execute(sql, params, truncate=True) diff --git a/tests/test_routes.py b/tests/test_routes.py index 6718c232..349ac302 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -12,26 +12,26 @@ def routes(): @pytest.mark.parametrize( "path,expected_class,expected_matches", ( - ("/", "IndexView", {"as_format": ""}), - ("/foo", "DatabaseView", {"as_format": None, "db_name": "foo"}), - ("/foo.csv", "DatabaseView", {"as_format": ".csv", "db_name": "foo"}), - ("/foo.json", "DatabaseView", {"as_format": ".json", "db_name": "foo"}), - ("/foo.humbug", "DatabaseView", {"as_format": None, "db_name": "foo.humbug"}), - ("/foo/humbug", "TableView", {"db_name": "foo", "table": "humbug"}), - ("/foo/humbug.json", "TableView", {"db_name": "foo", "table": "humbug"}), - ("/foo/humbug.blah", "TableView", {"db_name": "foo", "table": "humbug"}), + ("/", "IndexView", {"format": ""}), + ("/foo", "DatabaseView", {"format": None, "database": "foo"}), + ("/foo.csv", "DatabaseView", {"format": ".csv", "database": "foo"}), + ("/foo.json", "DatabaseView", {"format": ".json", "database": "foo"}), + ("/foo.humbug", "DatabaseView", {"format": None, "database": "foo.humbug"}), + ("/foo/humbug", "TableView", {"database": "foo", "table": "humbug"}), + ("/foo/humbug.json", "TableView", {"database": "foo", "table": "humbug"}), + ("/foo/humbug.blah", "TableView", {"database": "foo", "table": "humbug"}), ( "/foo/humbug/1", "RowView", - {"as_format": None, "db_name": "foo", "pk_path": "1", "table": "humbug"}, + {"format": None, "database": "foo", "pks": "1", "table": "humbug"}, ), ( "/foo/humbug/1.json", "RowView", - {"as_format": ".json", "db_name": "foo", "pk_path": "1", "table": "humbug"}, + {"format": ".json", "database": "foo", "pks": "1", "table": "humbug"}, ), - ("/-/metadata.json", "JsonDataView", {"as_format": ".json"}), - ("/-/metadata", "JsonDataView", {"as_format": ""}), + ("/-/metadata.json", "JsonDataView", {"format": ".json"}), + ("/-/metadata", "JsonDataView", {"format": ""}), ), ) def test_routes(routes, path, expected_class, expected_matches): From b9c2b1cfc8692b9700416db98721fa3ec982f6be Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 13:29:10 -0700 Subject: [PATCH 0651/1866] Consistent treatment of format in route capturing, refs #1667 Also refs #1660 --- datasette/app.py | 30 ++++++++++++------------------ tests/test_api.py | 4 ++-- tests/test_routes.py | 32 ++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 5259c50c..edef34e9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -985,10 +985,7 @@ class Datasette: def add_route(view, regex): routes.append((regex, view)) - # Generate a regex snippet to match all registered renderer file extensions - renderer_regex = "|".join(r"\." + key for key in self.renderers.keys()) - - add_route(IndexView.as_view(self), r"/(?P<format>(\.jsono?)?$)") + add_route(IndexView.as_view(self), r"/(\.(?P<format>jsono?))?$") # TODO: /favicon.ico and /-/static/ deserve far-future cache expires add_route(favicon, "/favicon.ico") @@ -1020,21 +1017,21 @@ class Datasette: ) add_route( JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()), - r"/-/metadata(?P<format>(\.json)?)$", + r"/-/metadata(\.(?P<format>json))?$", ) add_route( JsonDataView.as_view(self, "versions.json", self._versions), - r"/-/versions(?P<format>(\.json)?)$", + r"/-/versions(\.(?P<format>json))?$", ) add_route( JsonDataView.as_view( self, "plugins.json", self._plugins, needs_request=True ), - r"/-/plugins(?P<format>(\.json)?)$", + r"/-/plugins(\.(?P<format>json))?$", ) add_route( JsonDataView.as_view(self, "settings.json", lambda: self._settings), - r"/-/settings(?P<format>(\.json)?)$", + r"/-/settings(\.(?P<format>json))?$", ) add_route( permanent_redirect("/-/settings.json"), @@ -1046,15 +1043,15 @@ class Datasette: ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), - r"/-/threads(?P<format>(\.json)?)$", + r"/-/threads(\.(?P<format>json))?$", ) add_route( JsonDataView.as_view(self, "databases.json", self._connected_databases), - r"/-/databases(?P<format>(\.json)?)$", + r"/-/databases(\.(?P<format>json))?$", ) add_route( JsonDataView.as_view(self, "actor.json", self._actor, needs_request=True), - r"/-/actor(?P<format>(\.json)?)$", + r"/-/actor(\.(?P<format>json))?$", ) add_route( AuthTokenView.as_view(self), @@ -1080,20 +1077,17 @@ class Datasette: PatternPortfolioView.as_view(self), r"/-/patterns$", ) - add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^/]+?)\.db$") + add_route(DatabaseDownload.as_view(self), r"/(?P<database>[^\/\.]+)\.db$") add_route( - DatabaseView.as_view(self), - r"/(?P<database>[^/]+?)(?P<format>" + renderer_regex + r"|.jsono|\.csv)?$", + DatabaseView.as_view(self), r"/(?P<database>[^\/\.]+)(\.(?P<format>\w+))?$" ) add_route( TableView.as_view(self), - r"/(?P<database>[^/]+)/(?P<table>[^\/\.]+)(\.[a-zA-Z0-9_]+)?$", + r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$", ) add_route( RowView.as_view(self), - r"/(?P<database>[^/]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(?P<format>" - + renderer_regex - + r")?$", + r"/(?P<database>[^\/\.]+)/(?P<table>[^/]+?)/(?P<pks>[^/]+?)(\.(?P<format>\w+))?$", ) return [ # Compile any strings to regular expressions diff --git a/tests/test_api.py b/tests/test_api.py index 421bb1fe..253c1718 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -629,8 +629,8 @@ def test_old_memory_urls_redirect(app_client_no_files, path, expected_redirect): def test_database_page_for_database_with_dot_in_name(app_client_with_dot): - response = app_client_with_dot.get("/fixtures.dot.json") - assert 200 == response.status + response = app_client_with_dot.get("/fixtures~2Edot.json") + assert response.status == 200 def test_custom_sql(app_client): diff --git a/tests/test_routes.py b/tests/test_routes.py index 349ac302..1fa55018 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -12,14 +12,26 @@ def routes(): @pytest.mark.parametrize( "path,expected_class,expected_matches", ( - ("/", "IndexView", {"format": ""}), + ("/", "IndexView", {"format": None}), ("/foo", "DatabaseView", {"format": None, "database": "foo"}), - ("/foo.csv", "DatabaseView", {"format": ".csv", "database": "foo"}), - ("/foo.json", "DatabaseView", {"format": ".json", "database": "foo"}), - ("/foo.humbug", "DatabaseView", {"format": None, "database": "foo.humbug"}), - ("/foo/humbug", "TableView", {"database": "foo", "table": "humbug"}), - ("/foo/humbug.json", "TableView", {"database": "foo", "table": "humbug"}), - ("/foo/humbug.blah", "TableView", {"database": "foo", "table": "humbug"}), + ("/foo.csv", "DatabaseView", {"format": "csv", "database": "foo"}), + ("/foo.json", "DatabaseView", {"format": "json", "database": "foo"}), + ("/foo.humbug", "DatabaseView", {"format": "humbug", "database": "foo"}), + ( + "/foo/humbug", + "TableView", + {"database": "foo", "table": "humbug", "format": None}, + ), + ( + "/foo/humbug.json", + "TableView", + {"database": "foo", "table": "humbug", "format": "json"}, + ), + ( + "/foo/humbug.blah", + "TableView", + {"database": "foo", "table": "humbug", "format": "blah"}, + ), ( "/foo/humbug/1", "RowView", @@ -28,10 +40,10 @@ def routes(): ( "/foo/humbug/1.json", "RowView", - {"format": ".json", "database": "foo", "pks": "1", "table": "humbug"}, + {"format": "json", "database": "foo", "pks": "1", "table": "humbug"}, ), - ("/-/metadata.json", "JsonDataView", {"format": ".json"}), - ("/-/metadata", "JsonDataView", {"format": ""}), + ("/-/metadata.json", "JsonDataView", {"format": "json"}), + ("/-/metadata", "JsonDataView", {"format": None}), ), ) def test_routes(routes, path, expected_class, expected_matches): From 798f075ef9b98819fdb564f9f79c78975a0f71e8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 13:32:29 -0700 Subject: [PATCH 0652/1866] Read format from route captures, closes #1667 Refs #1660 --- datasette/utils/__init__.py | 20 -------------------- datasette/views/base.py | 12 +----------- tests/test_utils.py | 25 ------------------------- 3 files changed, 1 insertion(+), 56 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index ccdf8ad4..c89b9d23 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -731,26 +731,6 @@ def module_from_path(path, name): return mod -async def resolve_table_and_format( - table_and_format, table_exists, allowed_formats=None -): - if allowed_formats is None: - allowed_formats = [] - if "." in table_and_format: - # Check if a table exists with this exact name - it_exists = await table_exists(table_and_format) - if it_exists: - return table_and_format, None - - # Check if table ends with a known format - formats = list(allowed_formats) + ["csv", "jsono"] - for _format in formats: - if table_and_format.endswith(f".{_format}"): - table = table_and_format[: -(len(_format) + 1)] - return table, _format - return table_and_format, None - - def path_with_format( *, request=None, path=None, format=None, extra_qs=None, replace_format=None ): diff --git a/datasette/views/base.py b/datasette/views/base.py index 0bbf98bb..24e97d95 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -19,12 +19,10 @@ from datasette.utils import ( LimitedWriter, call_with_supported_arguments, tilde_decode, - tilde_encode, path_from_row_pks, path_with_added_args, path_with_removed_args, path_with_format, - resolve_table_and_format, sqlite3, HASH_LENGTH, ) @@ -372,18 +370,10 @@ class DataView(BaseView): return AsgiStream(stream_fn, headers=headers, content_type=content_type) - def get_format(self, request): - # Format is the bit from the path following the ., if one exists - last_path_component = request.path.split("/")[-1] - if "." in last_path_component: - return last_path_component.split(".")[-1] - else: - return None - async def get(self, request): db_name = request.url_vars["database"] database = tilde_decode(db_name) - _format = self.get_format(request) + _format = request.url_vars["format"] data_kwargs = {} if _format == "csv": diff --git a/tests/test_utils.py b/tests/test_utils.py index 790aadc7..7b41a87f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -351,31 +351,6 @@ def test_compound_keys_after_sql(): ) -async def table_exists(table): - return table == "exists.csv" - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "table_and_format,expected_table,expected_format", - [ - ("blah", "blah", None), - ("blah.csv", "blah", "csv"), - ("blah.json", "blah", "json"), - ("blah.baz", "blah.baz", None), - ("exists.csv", "exists.csv", None), - ], -) -async def test_resolve_table_and_format( - table_and_format, expected_table, expected_format -): - actual_table, actual_format = await utils.resolve_table_and_format( - table_and_format, table_exists, ["json"] - ) - assert expected_table == actual_table - assert expected_format == actual_format - - def test_table_columns(): conn = sqlite3.connect(":memory:") conn.executescript( From 7a6654a253dee243518dc542ce4c06dbb0d0801d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 17:11:17 -0700 Subject: [PATCH 0653/1866] Databases can now have a .route separate from their .name, refs #1668 --- datasette/app.py | 13 ++++++-- datasette/database.py | 1 + datasette/views/base.py | 12 +++++-- datasette/views/database.py | 18 ++++++----- datasette/views/table.py | 29 ++++++++++++----- docs/internals.rst | 11 ++++--- tests/test_internals_datasette.py | 1 + tests/test_routes.py | 52 ++++++++++++++++++++++++++++++- 8 files changed, 111 insertions(+), 26 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index edef34e9..5c8101a3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -388,13 +388,18 @@ class Datasette: def unsign(self, signed, namespace="default"): return URLSafeSerializer(self._secret, namespace).loads(signed) - def get_database(self, name=None): + def get_database(self, name=None, route=None): + if route is not None: + matches = [db for db in self.databases.values() if db.route == route] + if not matches: + raise KeyError + return matches[0] if name is None: - # Return first no-_schemas database + # Return first database that isn't "_internal" name = [key for key in self.databases.keys() if key != "_internal"][0] return self.databases[name] - def add_database(self, db, name=None): + def add_database(self, db, name=None, route=None): new_databases = self.databases.copy() if name is None: # Pick a unique name for this database @@ -407,6 +412,7 @@ class Datasette: name = "{}_{}".format(suggestion, i) i += 1 db.name = name + db.route = route or name new_databases[name] = db # don't mutate! that causes race conditions with live import self.databases = new_databases @@ -693,6 +699,7 @@ class Datasette: return [ { "name": d.name, + "route": d.route, "path": d.path, "size": d.size, "is_mutable": d.is_mutable, diff --git a/datasette/database.py b/datasette/database.py index 6ce87215..ba594a8c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -31,6 +31,7 @@ class Database: self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None ): self.name = None + self.route = None self.ds = ds self.path = path self.is_mutable = is_mutable diff --git a/datasette/views/base.py b/datasette/views/base.py index 24e97d95..afa9eaa6 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -371,13 +371,19 @@ class DataView(BaseView): return AsgiStream(stream_fn, headers=headers, content_type=content_type) async def get(self, request): - db_name = request.url_vars["database"] - database = tilde_decode(db_name) + database_route = tilde_decode(request.url_vars["database"]) + + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + _format = request.url_vars["format"] data_kwargs = {} if _format == "csv": - return await self.as_csv(request, database) + return await self.as_csv(request, database_route) if _format is None: # HTML views default to expanding all foreign key labels diff --git a/datasette/views/database.py b/datasette/views/database.py index 93bd1011..2563c5b2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -32,7 +32,13 @@ class DatabaseView(DataView): name = "database" async def data(self, request, default_labels=False, _size=None): - database = tilde_decode(request.url_vars["database"]) + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + await self.check_permissions( request, [ @@ -50,11 +56,6 @@ class DatabaseView(DataView): request, sql, _size=_size, metadata=metadata ) - try: - db = self.ds.databases[database] - except KeyError: - raise NotFound("Database not found: {}".format(database)) - table_counts = await db.table_counts(5) hidden_table_names = set(await db.hidden_table_names()) all_foreign_keys = await db.get_all_foreign_keys() @@ -171,9 +172,10 @@ class DatabaseDownload(DataView): "view-instance", ], ) - if database not in self.ds.databases: + try: + db = self.ds.get_database(route=database) + except KeyError: raise DatasetteError("Invalid database", status=404) - db = self.ds.databases[database] if db.is_memory: raise DatasetteError("Cannot download in-memory databases", status=404) if not self.ds.setting("allow_download") or db.is_mutable: diff --git a/datasette/views/table.py b/datasette/views/table.py index ea4f24b7..7fa1da3a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -272,10 +272,15 @@ class TableView(RowTableShared): name = "table" async def post(self, request): - db_name = tilde_decode(request.url_vars["database"]) + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name table = tilde_decode(request.url_vars["table"]) # Handle POST to a canned query - canned_query = await self.ds.get_canned_query(db_name, table, request.actor) + canned_query = await self.ds.get_canned_query(database, table, request.actor) assert canned_query, "You may only POST to a canned query" return await QueryView(self.ds).data( request, @@ -327,12 +332,13 @@ class TableView(RowTableShared): _next=None, _size=None, ): - database = tilde_decode(request.url_vars["database"]) + database_route = tilde_decode(request.url_vars["database"]) table = tilde_decode(request.url_vars["table"]) try: - db = self.ds.databases[database] + db = self.ds.get_database(route=database_route) except KeyError: - raise NotFound("Database not found: {}".format(database)) + raise NotFound("Database not found: {}".format(database_route)) + database = db.name # If this is a canned query, not a table, then dispatch to QueryView instead canned_query = await self.ds.get_canned_query(database, table, request.actor) @@ -938,8 +944,13 @@ class RowView(RowTableShared): name = "row" async def data(self, request, default_labels=False): - database = tilde_decode(request.url_vars["database"]) + database_route = tilde_decode(request.url_vars["database"]) table = tilde_decode(request.url_vars["table"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name await self.check_permissions( request, [ @@ -949,7 +960,11 @@ class RowView(RowTableShared): ], ) pk_values = urlsafe_components(request.url_vars["pks"]) - db = self.ds.databases[database] + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name sql, params, pks = await _sql_params_pks(db, table, pk_values) results = await db.execute(sql, params, truncate=True) columns = [r[0] for r in results.description] diff --git a/docs/internals.rst b/docs/internals.rst index 117cb95c..323256c7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -307,14 +307,17 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe .. _datasette_add_database: -.add_database(db, name=None) ----------------------------- +.add_database(db, name=None, route=None) +---------------------------------------- ``db`` - datasette.database.Database instance The database to be attached. ``name`` - string, optional - The name to be used for this database - this will be used in the URL path, e.g. ``/dbname``. If not specified Datasette will pick one based on the filename or memory name. + The name to be used for this database . If not specified Datasette will pick one based on the filename or memory name. + +``route`` - string, optional + This will be used in the URL path. If not specified, it will default to the same thing as the ``name``. The ``datasette.add_database(db)`` method lets you add a new database to the current Datasette instance. @@ -371,7 +374,7 @@ Using either of these pattern will result in the in-memory database being served ``name`` - string The name of the database to be removed. -This removes a database that has been previously added. ``name=`` is the unique name of that database, used in its URL path. +This removes a database that has been previously added. ``name=`` is the unique name of that database. .. _datasette_sign: diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index adf84be9..cc200a2d 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -55,6 +55,7 @@ async def test_datasette_constructor(): assert databases == [ { "name": "_memory", + "route": "_memory", "path": None, "size": 0, "is_mutable": False, diff --git a/tests/test_routes.py b/tests/test_routes.py index 1fa55018..dd3bc644 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1,6 +1,7 @@ -from datasette.app import Datasette +from datasette.app import Datasette, Database from datasette.utils import resolve_routes import pytest +import pytest_asyncio @pytest.fixture(scope="session") @@ -53,3 +54,52 @@ def test_routes(routes, path, expected_class, expected_matches): else: assert view.view_class.__name__ == expected_class assert match.groupdict() == expected_matches + + +@pytest_asyncio.fixture +async def ds_with_route(): + ds = Datasette() + ds.remove_database("_memory") + db = Database(ds, is_memory=True, memory_name="route-name-db") + ds.add_database(db, name="name", route="route-name") + await db.execute_write_script( + """ + create table if not exists t (id integer primary key); + insert or replace into t (id) values (1); + """ + ) + return ds + + +@pytest.mark.asyncio +async def test_db_with_route_databases(ds_with_route): + response = await ds_with_route.client.get("/-/databases.json") + assert response.json()[0] == { + "name": "name", + "route": "route-name", + "path": None, + "size": 0, + "is_mutable": True, + "is_memory": True, + "hash": None, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,expected_status", + ( + ("/", 200), + ("/name", 404), + ("/name/t", 404), + ("/name/t/1", 404), + ("/route-name", 200), + ("/route-name/t", 200), + ("/route-name/t/1", 200), + ), +) +async def test_db_with_route_that_does_not_match_name( + ds_with_route, path, expected_status +): + response = await ds_with_route.client.get(path) + assert response.status_code == expected_status From e10da9af3595c0a4e09c6f370103571aa4ea106e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 17:21:56 -0700 Subject: [PATCH 0654/1866] alternative-route demo, refs #1668 --- .github/workflows/deploy-latest.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 1ae96e89..92aa1c6b 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -42,6 +42,17 @@ jobs: sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. + - name: Set up the alternate-route demo + run: | + echo ' + from datasette import hookimpl + + @hookimpl + def startup(datasette): + db = datasette.get_database("fixtures2") + db.route = "alternative-route" + ' > plugins/alternative_route.py + cp fixtures.db fixtures2.db - name: Set up Cloud Run uses: google-github-actions/setup-gcloud@master with: @@ -54,7 +65,7 @@ jobs: gcloud config set project datasette-222320 export SUFFIX="-${GITHUB_REF#refs/heads/}" export SUFFIX=${SUFFIX#-main} - datasette publish cloudrun fixtures.db extra_database.db \ + datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ -m fixtures.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ From cdbae2b93f441653616dd889644c63e4150ceec1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 17:31:23 -0700 Subject: [PATCH 0655/1866] Fixed internal links to respect db.route, refs #1668 --- datasette/url_builder.py | 3 ++- datasette/views/table.py | 5 ++--- tests/test_routes.py | 22 +++++++++++++--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 498ec85d..574bf3c1 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -28,7 +28,8 @@ class Urls: return self.path("-/logout") def database(self, database, format=None): - return self.path(tilde_encode(database), format=format) + db = self.ds.get_database(database) + return self.path(tilde_encode(db.route), format=format) def table(self, database, table, format=None): path = f"{self.database(database)}/{tilde_encode(table)}" diff --git a/datasette/views/table.py b/datasette/views/table.py index 7fa1da3a..8745c28a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -141,10 +141,9 @@ class RowTableShared(DataView): "is_special_link_column": is_special_link_column, "raw": pk_path, "value": markupsafe.Markup( - '<a href="{base_url}{database}/{table}/{flat_pks_quoted}">{flat_pks}</a>'.format( + '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format( base_url=base_url, - database=database, - table=tilde_encode(table), + table_path=self.ds.urls.table(database, table), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) diff --git a/tests/test_routes.py b/tests/test_routes.py index dd3bc644..211b77b5 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -61,7 +61,7 @@ async def ds_with_route(): ds = Datasette() ds.remove_database("_memory") db = Database(ds, is_memory=True, memory_name="route-name-db") - ds.add_database(db, name="name", route="route-name") + ds.add_database(db, name="original-name", route="custom-route-name") await db.execute_write_script( """ create table if not exists t (id integer primary key); @@ -75,8 +75,8 @@ async def ds_with_route(): async def test_db_with_route_databases(ds_with_route): response = await ds_with_route.client.get("/-/databases.json") assert response.json()[0] == { - "name": "name", - "route": "route-name", + "name": "original-name", + "route": "custom-route-name", "path": None, "size": 0, "is_mutable": True, @@ -90,12 +90,12 @@ async def test_db_with_route_databases(ds_with_route): "path,expected_status", ( ("/", 200), - ("/name", 404), - ("/name/t", 404), - ("/name/t/1", 404), - ("/route-name", 200), - ("/route-name/t", 200), - ("/route-name/t/1", 200), + ("/original-name", 404), + ("/original-name/t", 404), + ("/original-name/t/1", 404), + ("/custom-route-name", 200), + ("/custom-route-name/t", 200), + ("/custom-route-name/t/1", 200), ), ) async def test_db_with_route_that_does_not_match_name( @@ -103,3 +103,7 @@ async def test_db_with_route_that_does_not_match_name( ): response = await ds_with_route.client.get(path) assert response.status_code == expected_status + # There should be links to custom-route-name but none to original-name + if response.status_code == 200: + assert "/custom-route-name" in response.text + assert "/original-name" not in response.text From 5471e3c4914837de957e206d8fb80c9ec383bc2e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 18:14:40 -0700 Subject: [PATCH 0656/1866] Release 0.61a0 Refs #957, #1533, #1545, #1576, #1577, #1587, #1601, #1603, #1607, #1612, #1621, #1649, #1654, #1657, #1661, #1668 --- datasette/version.py | 2 +- docs/changelog.rst | 29 +++++++++++++++++++++++++++-- docs/performance.rst | 2 ++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 91224615..ccc1e04b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.60.2" +__version__ = "0.61a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index c58c8444..0f3d3aff 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,14 +4,39 @@ Changelog ========= -.. _v0_60.2: +.. _v0_61_a0: + +0.61a0 (2022-03-19) +------------------- + +- Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`) +- Databases can now have a custom path within the Datasette instance that is indpendent of the database name, using the ``db.route`` property. (:issue:`1668`) +- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`) +- Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`) +- ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header. +- Canned queries are now shown at the top of the database page, directly below the SQL editor. Previously they were shown at the bottom, below the list of tables. (:issue:`1612`) +- Datasette now has a default favicon. (:issue:`1603`) +- ``sqlite_stat`` tables are now hidden by default. (:issue:`1587`) +- SpatiaLite tables ``data_licenses``, ``KNN`` and ``KNN2`` are now hidden by default. (:issue:`1601`) +- Python 3.6 is no longer supported. (:issue:`1577`) +- Tests now run against Python 3.11-dev. (:issue:`1621`) +- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`) +- SQL query tracing mechanism now works for queries executed in ``asyncio`` sub-tasks, such as those created by ``asyncio.gather()``. (:issue:`1576`) +- :ref:`internals_tracer` mechanism is now documented. +- Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`) +- ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`) +- Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__. +- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__) +- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`) + +.. _v0_60_2: 0.60.2 (2022-02-07) ------------------- - Fixed a bug where Datasette would open the same file twice with two different database names if you ran ``datasette file.db file.db``. (:issue:`1632`) -.. _v0_60.1: +.. _v0_60_1: 0.60.1 (2022-01-20) ------------------- diff --git a/docs/performance.rst b/docs/performance.rst index d37f1804..89bbf5ae 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -60,6 +60,8 @@ The :ref:`setting_default_cache_ttl` setting sets the default HTTP cache TTL for You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` query string parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query. +.. _performance_hashed_urls: + datasette-hashed-urls --------------------- From cb4854a435cc1418665edec2a73664ad74a32017 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 18:17:58 -0700 Subject: [PATCH 0657/1866] Fixed typo --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f3d3aff..9f5a143c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Changelog ------------------- - Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`) -- Databases can now have a custom path within the Datasette instance that is indpendent of the database name, using the ``db.route`` property. (:issue:`1668`) +- Databases can now have a custom path within the Datasette instance that is independent of the database name, using the ``db.route`` property. (:issue:`1668`) - URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`) - Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`) - ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header. From 4a4164b81191dec35e423486a208b05a9edc65e4 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 19 Mar 2022 18:23:03 -0700 Subject: [PATCH 0658/1866] Added another note to the 0.61a0 release notes, refs #1228 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9f5a143c..05ad85f2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ Changelog - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__. - Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__) - Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`) +- Fixed error caused when a table had a column named ``n``. (:issue:`1228`) .. _v0_60_2: From e627510b760198ccedba9e5af47a771e847785c9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 10:13:16 -0700 Subject: [PATCH 0659/1866] BaseView.check_permissions is now datasette.ensure_permissions, closes #1675 Refs #1660 --- datasette/app.py | 35 +++++++++++++++++++++++++++++++++++ datasette/views/base.py | 26 -------------------------- datasette/views/database.py | 12 ++++++------ datasette/views/table.py | 8 ++++---- docs/internals.rst | 26 ++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 36 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 5c8101a3..9e509e96 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,4 +1,5 @@ import asyncio +from typing import Sequence, Union, Tuple import asgi_csrf import collections import datetime @@ -628,6 +629,40 @@ class Datasette: ) return result + async def ensure_permissions( + self, + actor: dict, + permissions: Sequence[Union[Tuple[str, Union[str, Tuple[str, str]]], str]], + ): + """ + permissions is a list of (action, resource) tuples or 'action' strings + + Raises datasette.Forbidden() if any of the checks fail + """ + for permission in permissions: + if isinstance(permission, str): + action = permission + resource = None + elif isinstance(permission, (tuple, list)) and len(permission) == 2: + action, resource = permission + else: + assert ( + False + ), "permission should be string or tuple of two items: {}".format( + repr(permission) + ) + ok = await self.permission_allowed( + actor, + action, + resource=resource, + default=None, + ) + if ok is not None: + if ok: + return + else: + raise Forbidden(action) + async def execute( self, db_name, diff --git a/datasette/views/base.py b/datasette/views/base.py index afa9eaa6..d1e684a2 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -76,32 +76,6 @@ class BaseView: if not ok: raise Forbidden(action) - async def check_permissions(self, request, permissions): - """permissions is a list of (action, resource) tuples or 'action' strings""" - for permission in permissions: - if isinstance(permission, str): - action = permission - resource = None - elif isinstance(permission, (tuple, list)) and len(permission) == 2: - action, resource = permission - else: - assert ( - False - ), "permission should be string or tuple of two items: {}".format( - repr(permission) - ) - ok = await self.ds.permission_allowed( - request.actor, - action, - resource=resource, - default=None, - ) - if ok is not None: - if ok: - return - else: - raise Forbidden(action) - def database_color(self, database): return "ff0000" diff --git a/datasette/views/database.py b/datasette/views/database.py index 2563c5b2..69ed1233 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -39,8 +39,8 @@ class DatabaseView(DataView): raise NotFound("Database not found: {}".format(database_route)) database = db.name - await self.check_permissions( - request, + await self.ds.ensure_permissions( + request.actor, [ ("view-database", database), "view-instance", @@ -164,8 +164,8 @@ class DatabaseDownload(DataView): async def get(self, request): database = tilde_decode(request.url_vars["database"]) - await self.check_permissions( - request, + await self.ds.ensure_permissions( + request.actor, [ ("view-database-download", database), ("view-database", database), @@ -217,8 +217,8 @@ class QueryView(DataView): private = False if canned_query: # Respect canned query permissions - await self.check_permissions( - request, + await self.ds.ensure_permissions( + request.actor, [ ("view-query", (database, canned_query)), ("view-database", database), diff --git a/datasette/views/table.py b/datasette/views/table.py index 8745c28a..84169820 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -360,8 +360,8 @@ class TableView(RowTableShared): raise NotFound(f"Table not found: {table}") # Ensure user has permission to view this table - await self.check_permissions( - request, + await self.ds.ensure_permissions( + request.actor, [ ("view-table", (database, table)), ("view-database", database), @@ -950,8 +950,8 @@ class RowView(RowTableShared): except KeyError: raise NotFound("Database not found: {}".format(database_route)) database = db.name - await self.check_permissions( - request, + await self.ds.ensure_permissions( + request.actor, [ ("view-table", (database, table)), ("view-database", database), diff --git a/docs/internals.rst b/docs/internals.rst index 323256c7..12adde00 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -295,6 +295,32 @@ If neither ``metadata.json`` nor any of the plugins provide an answer to the per See :ref:`permissions` for a full list of permission actions included in Datasette core. +.. _datasette_permission_allowed: + +await .ensure_permissions(actor, permissions) +--------------------------------------------- + +``actor`` - dictionary + The authenticated actor. This is usually ``request.actor``. + +``permissions`` - list + A list of permissions to check. Each permission in that list can be a string ``action`` name or a 2-tuple of ``(action, resource)``. + +This method allows multiple permissions to be checked at onced. It raises a ``datasette.Forbidden`` exception if any of the checks are denied before one of them is explicitly granted. + +This is useful when you need to check multiple permissions at once. For example, an actor should be able to view a table if either one of the following checks returns ``True`` or not a single one of them returns ``False``: + +.. code-block:: python + + await self.ds.ensure_permissions( + request.actor, + [ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ] + ) + .. _datasette_get_database: .get_database(name) From dfafce6d962d615d98a7080e546c7b3662ae7d34 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 11:37:27 -0700 Subject: [PATCH 0660/1866] Display no-opinion permission checks on /-/permissions --- datasette/templates/permissions_debug.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index d898ea8c..db709c14 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -10,6 +10,9 @@ .check-result-false { color: red; } +.check-result-no-opinion { + color: #aaa; +} .check h2 { font-size: 1em } @@ -38,6 +41,8 @@ <span class="check-when">{{ check.when }}</span> {% if check.result %} <span class="check-result check-result-true">✓</span> + {% elif check.result is none %} + <span class="check-result check-result-no-opinion">none</span> {% else %} <span class="check-result check-result-false">✗</span> {% endif %} From 194e4f6c3fffde69eb196f8535ca45386b40ec2d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 11:41:56 -0700 Subject: [PATCH 0661/1866] Removed check_permission() from BaseView, closes #1677 Refs #1660 --- datasette/app.py | 1 + datasette/views/base.py | 10 ---------- datasette/views/database.py | 2 +- datasette/views/index.py | 2 +- datasette/views/special.py | 10 +++++----- tests/test_permissions.py | 13 ++++++++----- 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 9e509e96..22ae211f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -639,6 +639,7 @@ class Datasette: Raises datasette.Forbidden() if any of the checks fail """ + assert actor is None or isinstance(actor, dict) for permission in permissions: if isinstance(permission, str): action = permission diff --git a/datasette/views/base.py b/datasette/views/base.py index d1e684a2..221e1882 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -66,16 +66,6 @@ class BaseView: response.body = b"" return response - async def check_permission(self, request, action, resource=None): - ok = await self.ds.permission_allowed( - request.actor, - action, - resource=resource, - default=True, - ) - if not ok: - raise Forbidden(action) - def database_color(self, database): return "ff0000" diff --git a/datasette/views/database.py b/datasette/views/database.py index 69ed1233..31a1839f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -229,7 +229,7 @@ class QueryView(DataView): None, "view-query", (database, canned_query), default=True ) else: - await self.check_permission(request, "execute-sql", database) + await self.ds.ensure_permissions(request.actor, [("execute-sql", database)]) # Extract any :named parameters named_parameters = named_parameters or await derive_named_parameters( diff --git a/datasette/views/index.py b/datasette/views/index.py index f5e31181..1c391e26 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -20,7 +20,7 @@ class IndexView(BaseView): async def get(self, request): as_format = request.url_vars["format"] - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) databases = [] for name, db in self.ds.databases.items(): visible, database_private = await check_visibility( diff --git a/datasette/views/special.py b/datasette/views/special.py index 395ee587..dd834528 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -16,7 +16,7 @@ class JsonDataView(BaseView): async def get(self, request): as_format = request.url_vars["format"] - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) if self.needs_request: data = self.data_callback(request) else: @@ -47,7 +47,7 @@ class PatternPortfolioView(BaseView): has_json_alternate = False async def get(self, request): - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) return await self.render(["patterns.html"], request=request) @@ -95,7 +95,7 @@ class PermissionsDebugView(BaseView): has_json_alternate = False async def get(self, request): - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) if not await self.ds.permission_allowed(request.actor, "permissions-debug"): raise Forbidden("Permission denied") return await self.render( @@ -146,11 +146,11 @@ class MessagesDebugView(BaseView): has_json_alternate = False async def get(self, request): - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) return await self.render(["messages_debug.html"], request) async def post(self, request): - await self.check_permission(request, "view-instance") + await self.ds.ensure_permissions(request.actor, ["view-instance"]) post = await request.post_vars() message = post.get("message", "") message_type = post.get("message_type") or "INFO" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 788523b0..f4169dbe 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -321,17 +321,20 @@ def test_permissions_debug(app_client): checks = [ { "action": div.select_one(".check-action").text, - "result": bool(div.select(".check-result-true")), + # True = green tick, False = red cross, None = gray None + "result": None + if div.select(".check-result-no-opinion") + else bool(div.select(".check-result-true")), "used_default": bool(div.select(".check-used-default")), } for div in check_divs ] - assert [ + assert checks == [ {"action": "permissions-debug", "result": True, "used_default": False}, - {"action": "view-instance", "result": True, "used_default": True}, + {"action": "view-instance", "result": None, "used_default": True}, {"action": "permissions-debug", "result": False, "used_default": True}, - {"action": "view-instance", "result": True, "used_default": True}, - ] == checks + {"action": "view-instance", "result": None, "used_default": True}, + ] @pytest.mark.parametrize( From 1a7750eb29fd15dd2eea3b9f6e33028ce441b143 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 12:01:37 -0700 Subject: [PATCH 0662/1866] Documented datasette.check_visibility() method, closes #1678 --- datasette/app.py | 18 ++++++++++++++++++ datasette/utils/__init__.py | 19 ------------------- datasette/views/database.py | 10 +++------- datasette/views/index.py | 11 ++++------- docs/internals.rst | 28 +++++++++++++++++++++++++++- 5 files changed, 52 insertions(+), 34 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 22ae211f..c9eede26 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -664,6 +664,24 @@ class Datasette: else: raise Forbidden(action) + async def check_visibility(self, actor, action, resource): + """Returns (visible, private) - visible = can you see it, private = can others see it too""" + visible = await self.permission_allowed( + actor, + action, + resource=resource, + default=True, + ) + if not visible: + return False, False + private = not await self.permission_allowed( + None, + action, + resource=resource, + default=True, + ) + return visible, private + async def execute( self, db_name, diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c89b9d23..cd8e3d61 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1002,25 +1002,6 @@ def actor_matches_allow(actor, allow): return False -async def check_visibility(datasette, actor, action, resource, default=True): - """Returns (visible, private) - visible = can you see it, private = can others see it too""" - visible = await datasette.permission_allowed( - actor, - action, - resource=resource, - default=default, - ) - if not visible: - return False, False - private = not await datasette.permission_allowed( - None, - action, - resource=resource, - default=default, - ) - return visible, private - - def resolve_env_secrets(config, environ): """Create copy that recursively replaces {"$env": "NAME"} with values from environ""" if isinstance(config, dict): diff --git a/datasette/views/database.py b/datasette/views/database.py index 31a1839f..103bd575 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,6 @@ import markupsafe from datasette.utils import ( add_cors_headers, await_me_maybe, - check_visibility, derive_named_parameters, tilde_decode, to_css_class, @@ -62,8 +61,7 @@ class DatabaseView(DataView): views = [] for view_name in await db.view_names(): - visible, private = await check_visibility( - self.ds, + visible, private = await self.ds.check_visibility( request.actor, "view-table", (database, view_name), @@ -78,8 +76,7 @@ class DatabaseView(DataView): tables = [] for table in table_counts: - visible, private = await check_visibility( - self.ds, + visible, private = await self.ds.check_visibility( request.actor, "view-table", (database, table), @@ -105,8 +102,7 @@ class DatabaseView(DataView): for query in ( await self.ds.get_canned_queries(database, request.actor) ).values(): - visible, private = await check_visibility( - self.ds, + visible, private = await self.ds.check_visibility( request.actor, "view-query", (database, query["name"]), diff --git a/datasette/views/index.py b/datasette/views/index.py index 1c391e26..aec78814 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,7 +1,7 @@ import hashlib import json -from datasette.utils import add_cors_headers, check_visibility, CustomJSONEncoder +from datasette.utils import add_cors_headers, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ @@ -23,8 +23,7 @@ class IndexView(BaseView): await self.ds.ensure_permissions(request.actor, ["view-instance"]) databases = [] for name, db in self.ds.databases.items(): - visible, database_private = await check_visibility( - self.ds, + visible, database_private = await self.ds.check_visibility( request.actor, "view-database", name, @@ -36,8 +35,7 @@ class IndexView(BaseView): views = [] for view_name in await db.view_names(): - visible, private = await check_visibility( - self.ds, + visible, private = await self.ds.check_visibility( request.actor, "view-table", (name, view_name), @@ -55,8 +53,7 @@ class IndexView(BaseView): tables = {} for table in table_names: - visible, private = await check_visibility( - self.ds, + visible, private = await self.ds.check_visibility( request.actor, "view-table", (name, table), diff --git a/docs/internals.rst b/docs/internals.rst index 12adde00..f9a24fea 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -295,7 +295,7 @@ If neither ``metadata.json`` nor any of the plugins provide an answer to the per See :ref:`permissions` for a full list of permission actions included in Datasette core. -.. _datasette_permission_allowed: +.. _datasette_ensure_permissions: await .ensure_permissions(actor, permissions) --------------------------------------------- @@ -321,6 +321,32 @@ This is useful when you need to check multiple permissions at once. For example, ] ) +.. _datasette_check_visibilty: + +await .check_visibility(actor, action, resource=None) +----------------------------------------------------- + +``actor`` - dictionary + The authenticated actor. This is usually ``request.actor``. + +``action`` - string + The name of the action that is being permission checked. + +``resource`` - string or tuple, optional + The resource, e.g. the name of the database, or a tuple of two strings containing the name of the database and the name of the table. Only some permissions apply to a resource. + +This convenience method can be used to answer the question "should this item be considered private, in that it is visible to me but it is not visible to anonymous users?" + +It returns a tuple of two booleans, ``(visible, private)``. ``visible`` indicates if the actor can see this resource. ``private`` will be ``True`` if an anonymous user would not be able to view the resource. + +This example checks if the user can access a specific table, and sets ``private`` so that a padlock icon can later be displayed: + +.. code-block:: python + + visible, private = await self.ds.check_visibility( + request.actor, "view-table", (database, table) + ) + .. _datasette_get_database: .get_database(name) From 72bfd75fb7241893c931348e6aca712edc67ab04 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 14:55:50 -0700 Subject: [PATCH 0663/1866] Drop n=1 threshold down to <= 20ms, closes #1679 --- datasette/utils/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cd8e3d61..9109f823 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -182,15 +182,16 @@ class CustomJSONEncoder(json.JSONEncoder): def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) # n is the number of SQLite virtual machine instructions that will be - # executed between each check. It's hard to know what to pick here. - # After some experimentation, I've decided to go with 1000 by default and - # 1 for time limits that are less than 50ms + # executed between each check. It takes about 0.08ms to execute 1000. + # https://github.com/simonw/datasette/issues/1679 n = 1000 - if ms < 50: + if ms <= 20: + # This mainly happens while executing our test suite n = 1 def handler(): if time.perf_counter() >= deadline: + # Returning 1 terminates the query with an error return 1 conn.set_progress_handler(handler, n) From 12f3ca79956ed9003c874f67748432adcacc6fd2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 21 Mar 2022 18:42:03 -0700 Subject: [PATCH 0664/1866] google-github-actions/setup-gcloud@v0 --- .github/workflows/deploy-latest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 92aa1c6b..a61f6629 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - uses: actions/cache@v2 name: Configure pip caching with: @@ -54,7 +54,7 @@ jobs: ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: version: '275.0.0' service_account_email: ${{ secrets.GCP_SA_EMAIL }} From c4c9dbd0386e46d2bf199f0ed34e4895c98cb78c Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 22 Mar 2022 09:49:26 -0700 Subject: [PATCH 0665/1866] google-github-actions/setup-gcloud@v0 --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3cfc67da..3e4f8146 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -85,7 +85,7 @@ jobs: sphinx-to-sqlite ../docs.db _build cd .. - name: Set up Cloud Run - uses: google-github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@v0 with: version: '275.0.0' service_account_email: ${{ secrets.GCP_SA_EMAIL }} From d7c793d7998388d915f8d270079c68a77a785051 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 23 Mar 2022 11:12:26 -0700 Subject: [PATCH 0666/1866] Release 0.61 Refs #957, #1228, #1533, #1545, #1576, #1577, #1587, #1601, #1603, #1607, #1612, #1621, #1649, #1654, #1657, #1661, #1668, #1675, #1678 --- datasette/version.py | 2 +- docs/changelog.rst | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index ccc1e04b..f9b10696 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.61a0" +__version__ = "0.61" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 05ad85f2..d2de8da1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,30 +4,36 @@ Changelog ========= -.. _v0_61_a0: +.. _v0_61: -0.61a0 (2022-03-19) -------------------- +0.61 (2022-03-23) +----------------- +In preparation for Datasette 1.0, this release includes two potentially backwards-incompatible changes. Hashed URL mode has been moved to a separate plugin, and the way Datasette generates URLs to databases and tables with special characters in their name such as ``/`` and ``.`` has changed. + +Datasette also now requires Python 3.7 or higher. + +- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`) - Removed hashed URL mode from Datasette. The new ``datasette-hashed-urls`` plugin can be used to achieve the same result, see :ref:`performance_hashed_urls` for details. (:issue:`1661`) - Databases can now have a custom path within the Datasette instance that is independent of the database name, using the ``db.route`` property. (:issue:`1668`) -- URLs within Datasette now use a different encoding scheme for tables or databases that include "special" characters outside of the range of ``a-zA-Z0-9_-``. This scheme is explained here: :ref:`internals_tilde_encoding`. (:issue:`1657`) +- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`) +- Python 3.6 is no longer supported. (:issue:`1577`) +- Tests now run against Python 3.11-dev. (:issue:`1621`) +- New :ref:`datasette.ensure_permissions(actor, permissions) <datasette_ensure_permissions>` internal method for checking multiple permissions at once. (:issue:`1675`) +- New :ref:`datasette.check_visibility(actor, action, resource=None) <datasette_check_visibilty>` internal method for checking if a user can see a resource that would otherwise be invisible to unauthenticated users. (:issue:`1678`) - Table and row HTML pages now include a ``<link rel="alternate" type="application/json+datasette" href="...">`` element and return a ``Link: URL; rel="alternate"; type="application/json+datasette"`` HTTP header pointing to the JSON version of those pages. (:issue:`1533`) - ``Access-Control-Expose-Headers: Link`` is now added to the CORS headers, allowing remote JavaScript to access that header. - Canned queries are now shown at the top of the database page, directly below the SQL editor. Previously they were shown at the bottom, below the list of tables. (:issue:`1612`) - Datasette now has a default favicon. (:issue:`1603`) - ``sqlite_stat`` tables are now hidden by default. (:issue:`1587`) - SpatiaLite tables ``data_licenses``, ``KNN`` and ``KNN2`` are now hidden by default. (:issue:`1601`) -- Python 3.6 is no longer supported. (:issue:`1577`) -- Tests now run against Python 3.11-dev. (:issue:`1621`) -- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`) - SQL query tracing mechanism now works for queries executed in ``asyncio`` sub-tasks, such as those created by ``asyncio.gather()``. (:issue:`1576`) - :ref:`internals_tracer` mechanism is now documented. - Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`) - ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`) - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__. - Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__) -- Datasette is now covered by a `Code of Conduct <https://github.com/simonw/datasette/blob/main/CODE_OF_CONDUCT.md>`__. (:issue:`1654`) +- Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`) - Fixed error caused when a table had a column named ``n``. (:issue:`1228`) .. _v0_60_2: From 0159662ab8ccb363c59647861360e0cb7a6f930d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 23 Mar 2022 11:48:10 -0700 Subject: [PATCH 0667/1866] Fix for bug running ?sql= against databases with a different route, closes #1682 --- datasette/views/database.py | 7 ++++++- tests/test_routes.py | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 103bd575..bdd433cc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -203,7 +203,12 @@ class QueryView(DataView): named_parameters=None, write=False, ): - database = tilde_decode(request.url_vars["database"]) + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name params = {key: request.args.get(key) for key in request.args} if "sql" in params: params.pop("sql") diff --git a/tests/test_routes.py b/tests/test_routes.py index 211b77b5..5ae55d21 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -94,6 +94,7 @@ async def test_db_with_route_databases(ds_with_route): ("/original-name/t", 404), ("/original-name/t/1", 404), ("/custom-route-name", 200), + ("/custom-route-name?sql=select+id+from+t", 200), ("/custom-route-name/t", 200), ("/custom-route-name/t/1", 200), ), From d431a9055e977aefe48689a2e5866ea8d3558a6c Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 23 Mar 2022 11:54:10 -0700 Subject: [PATCH 0668/1866] Release 0.61.1 Refs #1682 Refs https://github.com/simonw/datasette-hashed-urls/issues/13 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index f9b10696..02451a1e 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.61" +__version__ = "0.61.1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d2de8da1..03cf62b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_61_1: + +0.61.1 (2022-03-23) +------------------- + +- Fixed a bug where databases with a different route from their name (as used by the `datasette-hashed-urls plugin <https://datasette.io/plugins/datasette-hashed-urls>`__) returned errors when executing custom SQL queries. (:issue:`1682`) + .. _v0_61: 0.61 (2022-03-23) From c496f2b663ff0cef908ffaaa68b8cb63111fb5f2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 24 Mar 2022 12:16:19 -0700 Subject: [PATCH 0669/1866] Don't show facet in cog menu if not allow_facet, closes #1683 --- datasette/static/table.js | 10 ++++++++-- datasette/templates/table.html | 1 + datasette/views/table.py | 3 +++ tests/test_table_html.py | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index 3c88cc40..096a27ac 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -128,7 +128,8 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig } else { hideColumn.parentNode.style.display = "none"; } - /* Only show facet if it's not the first column, not selected, not a single PK */ + /* Only show "Facet by this" if it's not the first column, not selected, + not a single PK and the Datasette allow_facet setting is True */ var displayedFacets = Array.from( document.querySelectorAll(".facet-info") ).map((el) => el.dataset.column); @@ -137,7 +138,12 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig var isSinglePk = th.getAttribute("data-is-pk") == "1" && document.querySelectorAll('th[data-is-pk="1"]').length == 1; - if (isFirstColumn || displayedFacets.includes(column) || isSinglePk) { + if ( + !DATASETTE_ALLOW_FACET || + isFirstColumn || + displayedFacets.includes(column) || + isSinglePk + ) { facetItem.parentNode.style.display = "none"; } else { facetItem.parentNode.style.display = "block"; diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 81bd044a..a9e88330 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -5,6 +5,7 @@ {% block extra_head %} {{- super() -}} <script src="{{ urls.static('table.js') }}" defer></script> +<script>DATASETTE_ALLOW_FACET = {{ datasette_allow_facet }};</script> <style> @media only screen and (max-width: 576px) { {% for column in display_columns -%} diff --git a/datasette/views/table.py b/datasette/views/table.py index 84169820..cd7afea6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -888,6 +888,9 @@ class TableView(RowTableShared): "metadata": metadata, "view_definition": await db.get_view_definition(table), "table_definition": await db.get_table_definition(table), + "datasette_allow_facet": "true" + if self.ds.setting("allow_facet") + else "false", } d.update(extra_context_from_filters) return d diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d40f017a..6dc26434 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1075,3 +1075,17 @@ def test_table_page_title(app_client, path, expected): response = app_client.get(path) title = Soup(response.text, "html.parser").find("title").text assert title == expected + + +@pytest.mark.parametrize("allow_facet", (True, False)) +def test_allow_facet_off(allow_facet): + with make_app_client(settings={"allow_facet": allow_facet}) as client: + response = client.get("/fixtures/facetable") + expected = "DATASETTE_ALLOW_FACET = {};".format( + "true" if allow_facet else "false" + ) + assert expected in response.text + if allow_facet: + assert "Suggested facets" in response.text + else: + assert "Suggested facets" not in response.text From 6b99e4a66ba0ed8fca8ee41ceb7206928b60d5d1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 25 Mar 2022 16:44:35 -0700 Subject: [PATCH 0670/1866] Added missing hookimpl import Useful for copying and pasting to create a quick plugin --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 92cf662f..9c1f4402 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -542,7 +542,7 @@ Return a list of ``(regex, view_function)`` pairs, something like this: .. code-block:: python - from datasette import Response + from datasette import hookimpl, Response import html From bd8a58ae61b2c986ef04ea721897906e0852e33e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 26 Mar 2022 13:51:20 -0700 Subject: [PATCH 0671/1866] Fix message_type in documentation, closes #1689 --- docs/internals.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index f9a24fea..0ba3fa69 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -458,8 +458,8 @@ Returns the original, decoded object that was passed to :ref:`datasette_sign`. I .. _datasette_add_message: -.add_message(request, message, message_type=datasette.INFO) ------------------------------------------------------------ +.add_message(request, message, type=datasette.INFO) +--------------------------------------------------- ``request`` - Request The current Request object @@ -467,7 +467,7 @@ Returns the original, decoded object that was passed to :ref:`datasette_sign`. I ``message`` - string The message string -``message_type`` - constant, optional +``type`` - constant, optional The message type - ``datasette.INFO``, ``datasette.WARNING`` or ``datasette.ERROR`` Datasette's flash messaging mechanism allows you to add a message that will be displayed to the user on the next page that they visit. Messages are persisted in a ``ds_messages`` cookie. This method adds a message to that cookie. From e73fa72917ca28c152208d62d07a490c81cadf52 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 26 Mar 2022 15:46:08 -0700 Subject: [PATCH 0672/1866] Fixed bug in httpx_mock example, closes #1691 --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 1291a875..8e4e3f91 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -198,7 +198,7 @@ Here's a test for that plugin that mocks the HTTPX outbound request: async def test_outbound_http_call(httpx_mock): httpx_mock.add_response( url='https://www.example.com/', - data='Hello world', + text='Hello world', ) datasette = Datasette([], memory=True) response = await datasette.client.post("/-/fetch-url", data={ From 5c5e9b365790d7c75cf2611e650d1013f587d316 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 31 Mar 2022 19:01:58 -0700 Subject: [PATCH 0673/1866] Request.fake(... url_vars), plus .fake() is now documented Also made 'from datasette import Request' shortcut work. Closes #1697 --- datasette/__init__.py | 2 +- datasette/utils/asgi.py | 4 +++- docs/internals.rst | 27 +++++++++++++++++++++++++++ tests/test_internals_request.py | 7 +++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/datasette/__init__.py b/datasette/__init__.py index faa36051..ea10c13d 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,5 +1,5 @@ from datasette.version import __version_info__, __version__ # noqa -from datasette.utils.asgi import Forbidden, NotFound, Response # noqa +from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from .hookspecs import hookimpl # noqa from .hookspecs import hookspec # noqa diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index cd3ec654..8a2fa060 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -118,7 +118,7 @@ class Request: return dict(parse_qsl(body.decode("utf-8"), keep_blank_values=True)) @classmethod - def fake(cls, path_with_query_string, method="GET", scheme="http"): + def fake(cls, path_with_query_string, method="GET", scheme="http", url_vars=None): """Useful for constructing Request objects for tests""" path, _, query_string = path_with_query_string.partition("?") scope = { @@ -130,6 +130,8 @@ class Request: "scheme": scheme, "type": "http", } + if url_vars: + scope["url_route"] = {"kwargs": url_vars} return cls(scope, None) diff --git a/docs/internals.rst b/docs/internals.rst index 0ba3fa69..854b96f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -60,6 +60,33 @@ The object also has two awaitable methods: ``await request.post_body()`` - bytes Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data. +And a class method that can be used to create fake request objects for use in tests: + +``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)`` + Returns a ``Request`` instance for the specified path and method. For example: + + .. code-block:: python + + from datasette import Request + from pprint import pprint + + request = Request.fake("/fixtures/facetable/", url_vars={ + "database": "fixtures", + "table": "facetable" + }) + pprint(request.scope) + + This outputs:: + + {'http_version': '1.1', + 'method': 'GET', + 'path': '/fixtures/facetable/', + 'query_string': b'', + 'raw_path': b'/fixtures/facetable/', + 'scheme': 'http', + 'type': 'http', + 'url_route': {'kwargs': {'database': 'fixtures', 'table': 'facetable'}}} + .. _internals_multiparams: The MultiParams class diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 44aaa153..d1ca1f46 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -75,6 +75,13 @@ def test_request_args(): request.args["missing"] +def test_request_fake_url_vars(): + request = Request.fake("/") + assert request.url_vars == {} + request = Request.fake("/", url_vars={"database": "fixtures"}) + assert request.url_vars == {"database": "fixtures"} + + def test_request_repr(): request = Request.fake("/foo?multi=1&multi=2&single=3") assert ( From df88d03298fa34d141ace7d6d8c35ca5e70576da Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 2 Apr 2022 23:05:10 -0700 Subject: [PATCH 0674/1866] Warn about Cloud Run and bots Refs #1698 --- docs/publish.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index 1d9664e7..166f2883 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -20,7 +20,14 @@ You will need a hosting account with `Heroku <https://www.heroku.com/>`__ or `Go Publishing to Google Cloud Run ------------------------------ -`Google Cloud Run <https://cloud.google.com/run/>`__ launched as a GA in in November 2019. It allows you to publish data in a scale-to-zero environment, so your application will start running when the first request is received and will shut down again when traffic ceases. This means you only pay for time spent serving traffic. +`Google Cloud Run <https://cloud.google.com/run/>`__ allows you to publish data in a scale-to-zero environment, so your application will start running when the first request is received and will shut down again when traffic ceases. This means you only pay for time spent serving traffic. + +.. warning:: + Cloud Run is a great option for inexpensively hosting small, low traffic projects - but costs can add up for projects that serve a lot of requests. + + Be particularly careful if your project has tables with large numbers of rows. Search engine crawlers that index a page for every row could result in a high bill. + + The `datasette-block-robots <https://datasette.io/plugins/datasette-block-robots>`__ plugin can be used to request search engine crawlers omit crawling your site, which can help avoid this issue. You will first need to install and configure the Google Cloud CLI tools by following `these instructions <https://cloud.google.com/sdk/>`__. @@ -171,4 +178,4 @@ You can customize the port that is exposed by the container using the ``--port`` A full list of options can be seen by running ``datasette package --help``: -See :ref:`cli_help_package___help` for the full list of options for this command. \ No newline at end of file +See :ref:`cli_help_package___help` for the full list of options for this command. From 90d1be9952db9aaddc21a536e4d00a8de44765d7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 6 Apr 2022 08:55:01 -0700 Subject: [PATCH 0675/1866] Tilde encoding now encodes space as plus, closes #1701 Refs #1657 --- datasette/utils/__init__.py | 12 ++++++++++-- docs/internals.rst | 6 ++++-- tests/test_html.py | 6 +++--- tests/test_utils.py | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 9109f823..4745254e 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1113,12 +1113,20 @@ _TILDE_ENCODING_SAFE = frozenset( # '.' and '~' ) +_space = ord(" ") + class TildeEncoder(dict): # Keeps a cache internally, via __missing__ def __missing__(self, b): + print("b is ", b) # Handle a cache miss, store encoded string in cache and return. - res = chr(b) if b in _TILDE_ENCODING_SAFE else "~{:02X}".format(b) + if b in _TILDE_ENCODING_SAFE: + res = chr(b) + elif b == _space: + res = "+" + else: + res = "~{:02X}".format(b) self[b] = res return res @@ -1138,7 +1146,7 @@ def tilde_decode(s: str) -> str: # Avoid accidentally decoding a %2f style sequence temp = secrets.token_hex(16) s = s.replace("%", temp) - decoded = urllib.parse.unquote(s.replace("~", "%")) + decoded = urllib.parse.unquote_plus(s.replace("~", "%")) return decoded.replace(temp, "%") diff --git a/docs/internals.rst b/docs/internals.rst index 854b96f8..76e27e5f 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -980,15 +980,17 @@ Datasette uses a custom encoding scheme in some places, called **tilde encoding* Tilde encoding uses the same algorithm as `URL percent-encoding <https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding>`__, but with the ``~`` tilde character used in place of ``%``. -Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example: +Any character other than ``ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz0123456789_-`` will be replaced by the numeric equivalent preceded by a tilde. For example: - ``/`` becomes ``~2F`` - ``.`` becomes ``~2E`` - ``%`` becomes ``~25`` - ``~`` becomes ``~7E`` -- Space character becomes ``~20`` +- Space becomes ``+`` - ``polls/2022.primary`` becomes ``polls~2F2022~2Eprimary`` +Note that the space character is a special case: it will be replaced with a ``+`` symbol. + .. _internals_utils_tilde_encode: .. autofunction:: datasette.utils.tilde_encode diff --git a/tests/test_html.py b/tests/test_html.py index 6e4c22b1..42f1a3ee 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -28,7 +28,7 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": "/extra~20database", "text": "extra database"}, + {"href": "/extra+database", "text": "extra database"}, {"href": "/fixtures", "text": "fixtures"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # Database should show count text and attached tables @@ -43,8 +43,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": r"/extra~20database/searchable", "text": "searchable"}, - {"href": r"/extra~20database/searchable_view", "text": "searchable_view"}, + {"href": r"/extra+database/searchable", "text": "searchable"}, + {"href": r"/extra+database/searchable_view", "text": "searchable_view"}, ] == table_links diff --git a/tests/test_utils.py b/tests/test_utils.py index 7b41a87f..df788767 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -618,6 +618,7 @@ async def test_derive_named_parameters(sql, expected): ("-/db-/table.csv", "-~2Fdb-~2Ftable~2Ecsv"), (r"%~-/", "~25~7E-~2F"), ("~25~7E~2D~2F", "~7E25~7E7E~7E2D~7E2F"), + ("with space", "with+space"), ), ) def test_tilde_encoding(original, expected): From 247e460e08bf823142f7b84058fe44e43626787f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:51:04 -0700 Subject: [PATCH 0676/1866] Update beautifulsoup4 requirement (#1703) Updates the requirements on [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/bs4/) to permit the latest version. --- updated-dependencies: - dependency-name: beautifulsoup4 dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b58b8c4..77fca8cd 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.17,<0.19", - "beautifulsoup4>=4.8.1,<4.11.0", + "beautifulsoup4>=4.8.1,<4.12.0", "black==22.1.0", "pytest-timeout>=1.4.2,<2.2", "trustme>=0.7,<0.10", From 138e4d9a53e3982137294ba383303c3a848cfca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Apr 2022 16:05:09 -0700 Subject: [PATCH 0677/1866] Update click requirement from <8.1.0,>=7.1.1 to >=7.1.1,<8.2.0 (#1694) Updates the requirements on [click](https://github.com/pallets/click) to permit the latest version. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/7.1.1...8.1.0) --- updated-dependencies: - dependency-name: click dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 77fca8cd..e5dd55fd 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( python_requires=">=3.7", install_requires=[ "asgiref>=3.2.10,<3.6.0", - "click>=7.1.1,<8.1.0", + "click>=7.1.1,<8.2.0", "click-default-group~=1.2.2", "Jinja2>=2.10.3,<3.1.0", "hupper~=1.9", From 143c105f875f4c8d4512233fa856477a938b38ca Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 Apr 2022 11:43:32 -0700 Subject: [PATCH 0678/1866] Removed rogue print --- datasette/utils/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 4745254e..77768112 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1119,7 +1119,6 @@ _space = ord(" ") class TildeEncoder(dict): # Keeps a cache internally, via __missing__ def __missing__(self, b): - print("b is ", b) # Handle a cache miss, store encoded string in cache and return. if b in _TILDE_ENCODING_SAFE: res = chr(b) From 0bc5186b7bb4fc82392df08f99a9132f84dcb331 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 Apr 2022 11:44:12 -0700 Subject: [PATCH 0679/1866] Tooltip and commas for byte length display, closes #1712 --- datasette/views/database.py | 12 +++++++++--- datasette/views/table.py | 8 +++++++- tests/test_table_html.py | 26 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index bdd433cc..9a8aca32 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, derive_named_parameters, + format_bytes, tilde_decode, to_css_class, validate_sql_select, @@ -399,13 +400,18 @@ class QueryView(DataView): ).hexdigest(), }, ) - display_value = Markup( - '<a class="blob-download" href="{}"><Binary: {} byte{}></a>'.format( + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format( blob_url, - len(display_value), + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), "" if len(value) == 1 else "s", ) ) + display_row.append(display_value) display_rows.append(display_row) diff --git a/datasette/views/table.py b/datasette/views/table.py index cd7afea6..dc85165e 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -12,10 +12,12 @@ from datasette.utils import ( MultiParams, append_querystring, compound_keys_after_sql, + format_bytes, tilde_decode, tilde_encode, escape_sqlite, filters_should_redirect, + format_bytes, is_url, path_from_row_pks, path_with_added_args, @@ -175,14 +177,18 @@ class RowTableShared(DataView): if plugin_display_value: display_value = plugin_display_value elif isinstance(value, bytes): + formatted = format_bytes(len(value)) display_value = markupsafe.Markup( - '<a class="blob-download" href="{}"><Binary: {} byte{}></a>'.format( + '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format( self.ds.urls.row_blob( database, table, path_from_row_pks(row, pks, not pks), column, ), + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", len(value), "" if len(value) == 1 else "s", ) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 6dc26434..d3cb3e17 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1,3 +1,4 @@ +from datasette.app import Datasette, Database from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, @@ -1089,3 +1090,28 @@ def test_allow_facet_off(allow_facet): assert "Suggested facets" in response.text else: assert "Suggested facets" not in response.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "size,title,length_bytes", + ( + (2000, ' title="2.0 KB"', "2,000"), + (20000, ' title="19.5 KB"', "20,000"), + (20, "", "20"), + ), +) +async def test_format_of_binary_links(size, title, length_bytes): + ds = Datasette() + db_name = "binary-links-{}".format(size) + db = ds.add_memory_database(db_name) + sql = "select zeroblob({}) as blob".format(size) + await db.execute_write("create table blobs as {}".format(sql)) + response = await ds.client.get("/{}/blobs".format(db_name)) + assert response.status_code == 200 + expected = "{}><Binary: {} bytes></a>".format(title, length_bytes) + assert expected in response.text + # And test with arbitrary SQL query too + sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql}) + assert sql_response.status_code == 200 + assert expected in sql_response.text From 8338c66a57502ef27c3d7afb2527fbc0663b2570 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 21 Apr 2022 11:05:43 -0700 Subject: [PATCH 0680/1866] datasette-geojson is an example of register_output_renderer --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9c1f4402..67842fc4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -526,7 +526,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th def can_render_demo(columns): return {"atom_id", "atom_title", "atom_updated"}.issubset(columns) -Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_ +Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__ .. _plugin_register_routes: From d57c347f35bcd8cff15f913da851b4b8eb030867 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 22 Apr 2022 14:58:46 -0700 Subject: [PATCH 0681/1866] Ignore Black commits in git blame, refs #1716 --- .git-blame-ignore-revs | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..84e574fd --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,4 @@ +# Applying Black +35d6ee2790e41e96f243c1ff58be0c9c0519a8ce +368638555160fb9ac78f462d0f79b1394163fa30 +2b344f6a34d2adaa305996a1a580ece06397f6e4 From 3001e1e394b6cb605c2cd81eed671a7da419c1b3 Mon Sep 17 00:00:00 2001 From: Tim Sherratt <tim@discontents.com.au> Date: Mon, 25 Apr 2022 00:03:08 +1000 Subject: [PATCH 0682/1866] Add timeout option to Cloudrun build (#1717) * Add timeout option for build phase * Make the --timeout setting optional * Add test for --timeout setting Thanks, @wragge --- datasette/publish/cloudrun.py | 12 +++++++++++- tests/test_publish_cloudrun.py | 31 +++++++++++++++++++------------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index a1e2f580..11a39fb2 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -41,6 +41,10 @@ def publish_subcommand(publish): type=click.Choice(["1", "2", "4"]), help="Number of vCPUs to allocate in Cloud Run", ) + @click.option( + "--timeout", + help="Build timeout in seconds", + ) @click.option( "--apt-get-install", "apt_get_extras", @@ -72,6 +76,7 @@ def publish_subcommand(publish): show_files, memory, cpu, + timeout, apt_get_extras, ): "Publish databases to Datasette running on Cloud Run" @@ -156,7 +161,12 @@ def publish_subcommand(publish): print("\n====================\n") image_id = f"gcr.io/{project}/{name}" - check_call(f"gcloud builds submit --tag {image_id}", shell=True) + check_call( + "gcloud builds submit --tag {}{}".format( + image_id, " --timeout {}".format(timeout) if timeout else "" + ), + shell=True, + ) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format( image_id, diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 9c8c38cf..3427f4f7 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -105,18 +105,19 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @pytest.mark.parametrize( - "memory,cpu,expected_gcloud_args", + "memory,cpu,timeout,expected_gcloud_args", [ - ["1Gi", None, "--memory 1Gi"], - ["2G", None, "--memory 2G"], - ["256Mi", None, "--memory 256Mi"], - ["4", None, None], - ["GB", None, None], - [None, 1, "--cpu 1"], - [None, 2, "--cpu 2"], - [None, 3, None], - [None, 4, "--cpu 4"], - ["2G", 4, "--memory 2G --cpu 4"], + ["1Gi", None, None, "--memory 1Gi"], + ["2G", None, None, "--memory 2G"], + ["256Mi", None, None, "--memory 256Mi"], + ["4", None, None, None], + ["GB", None, None, None], + [None, 1, None, "--cpu 1"], + [None, 2, None, "--cpu 2"], + [None, 3, None, None], + [None, 4, None, "--cpu 4"], + ["2G", 4, None, "--memory 2G --cpu 4"], + [None, None, 1800, "--timeout 1800"], ], ) def test_publish_cloudrun_memory_cpu( @@ -125,6 +126,7 @@ def test_publish_cloudrun_memory_cpu( mock_which, memory, cpu, + timeout, expected_gcloud_args, tmp_path_factory, ): @@ -139,6 +141,8 @@ def test_publish_cloudrun_memory_cpu( args.extend(["--memory", memory]) if cpu: args.extend(["--cpu", str(cpu)]) + if timeout: + args.extend(["--timeout", str(timeout)]) result = runner.invoke(cli.cli, args) if expected_gcloud_args is None: assert 2 == result.exit_code @@ -149,13 +153,16 @@ def test_publish_cloudrun_memory_cpu( "gcloud run deploy --allow-unauthenticated --platform=managed" " --image {} test".format(tag) ) + expected_build_call = f"gcloud builds submit --tag {tag}" if memory: expected_call += " --memory {}".format(memory) if cpu: expected_call += " --cpu {}".format(cpu) + if timeout: + expected_build_call += f" --timeout {timeout}" mock_call.assert_has_calls( [ - mock.call(f"gcloud builds submit --tag {tag}", shell=True), + mock.call(expected_build_call, shell=True), mock.call( expected_call, shell=True, From 4bd3a30e1ea460e17011c11c16408300b87d1106 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 07:04:11 -0700 Subject: [PATCH 0683/1866] Update cog docs for publish cloudrun, refs #1717 --- docs/cli-reference.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 69670d8a..3ca48aa2 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -248,6 +248,7 @@ datasette publish cloudrun --help metadata.json --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run + --timeout TEXT Build timeout in seconds --apt-get-install TEXT Additional packages to apt-get install --help Show this message and exit. From e64d14e413a955a10df88e106a8b5f1572ec8613 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 07:09:08 -0700 Subject: [PATCH 0684/1866] Use type integer for --timeout, refs #1717 --- datasette/publish/cloudrun.py | 1 + docs/cli-reference.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 11a39fb2..50b2b2fd 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -43,6 +43,7 @@ def publish_subcommand(publish): ) @click.option( "--timeout", + type=int, help="Build timeout in seconds", ) @click.option( diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3ca48aa2..2a6fbfc8 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -248,7 +248,7 @@ datasette publish cloudrun --help metadata.json --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run - --timeout TEXT Build timeout in seconds + --timeout INTEGER Build timeout in seconds --apt-get-install TEXT Additional packages to apt-get install --help Show this message and exit. From 40ef8ebac2d83c34f467fd2d7bf80f0549b6f6c3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 07:10:13 -0700 Subject: [PATCH 0685/1866] Run tests on pull requests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 478e1f34..c11bfa2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: Test -on: [push] +on: [push, pull_request] jobs: test: From 36573638b0948174ae237d62e6369b7d55220d7f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 08:50:43 -0700 Subject: [PATCH 0686/1866] Apply Black to code examples in documentation, refs #1718 Uses blacken-docs. This has a deliberate error which I hope will fail CI. --- .github/workflows/test.yml | 5 ++++ docs/contributing.rst | 9 +++++++ docs/spatialite.rst | 50 ++++++++++++++++++++++------------ docs/writing_plugins.rst | 55 ++++++++++++++++++++++---------------- setup.py | 3 ++- 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c11bfa2e..38b62995 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,3 +32,8 @@ jobs: - name: Check if cog needs to be run run: | cog --check docs/*.rst + - name: Check if blacken-docs needs to be run + run: | + blacken-docs -l 60 docs/*.rst + # This fails if a diff was generated: + git diff-index --quiet HEAD -- diff --git a/docs/contributing.rst b/docs/contributing.rst index b74f2f36..c193ba49 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -144,6 +144,15 @@ If any of your code does not conform to Black you can run this to automatically All done! ✨ 🍰 ✨ 1 file reformatted, 94 files left unchanged. +.. _contributing_formatting_blacken_docs: + +blacken-docs +~~~~~~~~~~~~ + +The `blacken-docs <https://pypi.org/project/blacken-docs/>`__ command applies Black formatting rules to code examples in the documentation. Run it like this:: + + blacken-docs -l 60 docs/*.rst + .. _contributing_formatting_prettier: Prettier diff --git a/docs/spatialite.rst b/docs/spatialite.rst index d1b300b2..52b6747e 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -58,21 +58,28 @@ Here's a recipe for taking a table with existing latitude and longitude columns, .. code-block:: python import sqlite3 - conn = sqlite3.connect('museums.db') + + conn = sqlite3.connect("museums.db") # Lead the spatialite extension: conn.enable_load_extension(True) - conn.load_extension('/usr/local/lib/mod_spatialite.dylib') + conn.load_extension("/usr/local/lib/mod_spatialite.dylib") # Initialize spatial metadata for this database: - conn.execute('select InitSpatialMetadata(1)') + conn.execute("select InitSpatialMetadata(1)") # Add a geometry column called point_geom to our museums table: - conn.execute("SELECT AddGeometryColumn('museums', 'point_geom', 4326, 'POINT', 2);") + conn.execute( + "SELECT AddGeometryColumn('museums', 'point_geom', 4326, 'POINT', 2);" + ) # Now update that geometry column with the lat/lon points - conn.execute(''' + conn.execute( + """ UPDATE museums SET point_geom = GeomFromText('POINT('||"longitude"||' '||"latitude"||')',4326); - ''') + """ + ) # Now add a spatial index to that column - conn.execute('select CreateSpatialIndex("museums", "point_geom");') + conn.execute( + 'select CreateSpatialIndex("museums", "point_geom");' + ) # If you don't commit your changes will not be persisted: conn.commit() conn.close() @@ -186,28 +193,37 @@ Here's Python code to create a SQLite database, enable SpatiaLite, create a plac .. code-block:: python import sqlite3 - conn = sqlite3.connect('places.db') + + conn = sqlite3.connect("places.db") # Enable SpatialLite extension conn.enable_load_extension(True) - conn.load_extension('/usr/local/lib/mod_spatialite.dylib') + conn.load_extension("/usr/local/lib/mod_spatialite.dylib") # Create the masic countries table - conn.execute('select InitSpatialMetadata(1)') - conn.execute('create table places (id integer primary key, name text);') + conn.execute("select InitSpatialMetadata(1)") + conn.execute( + "create table places (id integer primary key, name text);" + ) # Add a MULTIPOLYGON Geometry column - conn.execute("SELECT AddGeometryColumn('places', 'geom', 4326, 'MULTIPOLYGON', 2);") + conn.execute( + "SELECT AddGeometryColumn('places', 'geom', 4326, 'MULTIPOLYGON', 2);" + ) # Add a spatial index against the new column conn.execute("SELECT CreateSpatialIndex('places', 'geom');") # Now populate the table from shapely.geometry.multipolygon import MultiPolygon from shapely.geometry import shape import requests - geojson = requests.get('https://data.whosonfirst.org/404/227/475/404227475.geojson').json() + + geojson = requests.get( + "https://data.whosonfirst.org/404/227/475/404227475.geojson" + ).json() # Convert to "Well Known Text" format - wkt = shape(geojson['geometry']).wkt + wkt = shape(geojson["geometry"]).wkt # Insert and commit the record - conn.execute("INSERT INTO places (id, name, geom) VALUES(null, ?, GeomFromText(?, 4326))", ( - "Wales", wkt - )) + conn.execute( + "INSERT INTO places (id, name, geom) VALUES(null, ?, GeomFromText(?, 4326))", + ("Wales", wkt), + ) conn.commit() Querying polygons using within() diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index bd60a4b6..89f7f5eb 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -18,9 +18,12 @@ The quickest way to start writing a plugin is to create a ``my_plugin.py`` file from datasette import hookimpl + @hookimpl def prepare_connection(conn): - conn.create_function('hello_world', 0, lambda: 'Hello world!') + conn.create_function( + "hello_world", 0, lambda: "Hello world!" + ) If you save this in ``plugins/my_plugin.py`` you can then start Datasette like this:: @@ -60,22 +63,22 @@ The example consists of two files: a ``setup.py`` file that defines the plugin: from setuptools import setup - VERSION = '0.1' + VERSION = "0.1" setup( - name='datasette-plugin-demos', - description='Examples of plugins for Datasette', - author='Simon Willison', - url='https://github.com/simonw/datasette-plugin-demos', - license='Apache License, Version 2.0', + name="datasette-plugin-demos", + description="Examples of plugins for Datasette", + author="Simon Willison", + url="https://github.com/simonw/datasette-plugin-demos", + license="Apache License, Version 2.0", version=VERSION, - py_modules=['datasette_plugin_demos'], + py_modules=["datasette_plugin_demos"], entry_points={ - 'datasette': [ - 'plugin_demos = datasette_plugin_demos' + "datasette": [ + "plugin_demos = datasette_plugin_demos" ] }, - install_requires=['datasette'] + install_requires=["datasette"], ) And a Python module file, ``datasette_plugin_demos.py``, that implements the plugin: @@ -88,12 +91,14 @@ And a Python module file, ``datasette_plugin_demos.py``, that implements the plu @hookimpl def prepare_jinja2_environment(env): - env.filters['uppercase'] = lambda u: u.upper() + env.filters["uppercase"] = lambda u: u.upper() @hookimpl def prepare_connection(conn): - conn.create_function('random_integer', 2, random.randint) + conn.create_function( + "random_integer", 2, random.randint + ) Having built a plugin in this way you can turn it into an installable package using the following command:: @@ -123,11 +128,13 @@ To bundle the static assets for a plugin in the package that you publish to PyPI .. code-block:: python - package_data={ - 'datasette_plugin_name': [ - 'static/plugin.js', - ], - }, + package_data = ( + { + "datasette_plugin_name": [ + "static/plugin.js", + ], + }, + ) Where ``datasette_plugin_name`` is the name of the plugin package (note that it uses underscores, not hyphens) and ``static/plugin.js`` is the path within that package to the static file. @@ -152,11 +159,13 @@ Templates should be bundled for distribution using the same ``package_data`` mec .. code-block:: python - package_data={ - 'datasette_plugin_name': [ - 'templates/my_template.html', - ], - }, + package_data = ( + { + "datasette_plugin_name": [ + "templates/my_template.html", + ], + }, + ) You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema <https://github.com/simonw/datasette-edit-schema>`__ for an example of this pattern. diff --git a/setup.py b/setup.py index e5dd55fd..7f0562fd 100644 --- a/setup.py +++ b/setup.py @@ -65,13 +65,14 @@ setup( """, setup_requires=["pytest-runner"], extras_require={ - "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell", "blacken-docs"], "test": [ "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.17,<0.19", "beautifulsoup4>=4.8.1,<4.12.0", "black==22.1.0", + "blacken-docs==1.12.1", "pytest-timeout>=1.4.2,<2.2", "trustme>=0.7,<0.10", "cogapp>=3.3.0", From 92b26673d86a663050c9a40a8ffd5b56c25be85f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 08:51:09 -0700 Subject: [PATCH 0687/1866] Fix blacken-docs errors and warnings, refs #1718 --- docs/authentication.rst | 25 ++-- docs/internals.rst | 98 ++++++++----- docs/json_api.rst | 2 +- docs/plugin_hooks.rst | 305 +++++++++++++++++++++++++++------------- 4 files changed, 289 insertions(+), 141 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 0d98cf82..24960733 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -381,11 +381,10 @@ Authentication plugins can set signed ``ds_actor`` cookies themselves like so: .. code-block:: python response = Response.redirect("/") - response.set_cookie("ds_actor", datasette.sign({ - "a": { - "id": "cleopaws" - } - }, "actor")) + response.set_cookie( + "ds_actor", + datasette.sign({"a": {"id": "cleopaws"}}, "actor"), + ) Note that you need to pass ``"actor"`` as the namespace to :ref:`datasette_sign`. @@ -412,12 +411,16 @@ To include an expiry, add a ``"e"`` key to the cookie value containing a `base62 expires_at = int(time.time()) + (24 * 60 * 60) response = Response.redirect("/") - response.set_cookie("ds_actor", datasette.sign({ - "a": { - "id": "cleopaws" - }, - "e": baseconv.base62.encode(expires_at), - }, "actor")) + response.set_cookie( + "ds_actor", + datasette.sign( + { + "a": {"id": "cleopaws"}, + "e": baseconv.base62.encode(expires_at), + }, + "actor", + ), + ) The resulting cookie will encode data that looks something like this: diff --git a/docs/internals.rst b/docs/internals.rst index 76e27e5f..aad608dc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -70,10 +70,10 @@ And a class method that can be used to create fake request objects for use in te from datasette import Request from pprint import pprint - request = Request.fake("/fixtures/facetable/", url_vars={ - "database": "fixtures", - "table": "facetable" - }) + request = Request.fake( + "/fixtures/facetable/", + url_vars={"database": "fixtures", "table": "facetable"}, + ) pprint(request.scope) This outputs:: @@ -146,7 +146,7 @@ For example: response = Response( "<xml>This is XML</xml>", - content_type="application/xml; charset=utf-8" + content_type="application/xml; charset=utf-8", ) The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: @@ -157,9 +157,13 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp html_response = Response.html("This is HTML") json_response = Response.json({"this_is": "json"}) - text_response = Response.text("This will become utf-8 encoded text") + text_response = Response.text( + "This will become utf-8 encoded text" + ) # Redirects are served as 302, unless you pass status=301: - redirect_response = Response.redirect("https://latest.datasette.io/") + redirect_response = Response.redirect( + "https://latest.datasette.io/" + ) Each of these responses will use the correct corresponding content-type - ``text/html; charset=utf-8``, ``application/json; charset=utf-8`` or ``text/plain; charset=utf-8`` respectively. @@ -207,13 +211,17 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The httponly=False, samesite="lax", ): + ... You can use this with :ref:`datasette.sign() <datasette_sign>` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie <authentication_ds_actor>` for use with Datasette :ref:`authentication <authentication>`: .. code-block:: python response = Response.redirect("/") - response.set_cookie("ds_actor", datasette.sign({"a": {"id": "cleopaws"}}, "actor")) + response.set_cookie( + "ds_actor", + datasette.sign({"a": {"id": "cleopaws"}}, "actor"), + ) return response .. _internals_datasette: @@ -236,13 +244,16 @@ You can create your own instance of this - for example to help write tests for a datasette = Datasette(files=["/path/to/my-database.db"]) # Pass metadata as a JSON dictionary like this - datasette = Datasette(files=["/path/to/my-database.db"], metadata={ - "databases": { - "my-database": { - "description": "This is my database" + datasette = Datasette( + files=["/path/to/my-database.db"], + metadata={ + "databases": { + "my-database": { + "description": "This is my database" + } } - } - }) + }, + ) Constructor parameters include: @@ -345,7 +356,7 @@ This is useful when you need to check multiple permissions at once. For example, ("view-table", (database, table)), ("view-database", database), "view-instance", - ] + ], ) .. _datasette_check_visibilty: @@ -406,11 +417,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` from datasette.database import Database - datasette.add_database(Database( - datasette, - path="path/to/my-new-database.db", - is_mutable=True - )) + datasette.add_database( + Database( + datasette, + path="path/to/my-new-database.db", + is_mutable=True, + ) + ) This will add a mutable database and serve it at ``/my-new-database``. @@ -418,8 +431,12 @@ This will add a mutable database and serve it at ``/my-new-database``. .. code-block:: python - db = datasette.add_database(Database(datasette, memory_name="statistics")) - await db.execute_write("CREATE TABLE foo(id integer primary key)") + db = datasette.add_database( + Database(datasette, memory_name="statistics") + ) + await db.execute_write( + "CREATE TABLE foo(id integer primary key)" + ) .. _datasette_add_memory_database: @@ -438,10 +455,9 @@ This is a shortcut for the following: from datasette.database import Database - datasette.add_database(Database( - datasette, - memory_name="statistics" - )) + datasette.add_database( + Database(datasette, memory_name="statistics") + ) Using either of these pattern will result in the in-memory database being served at ``/statistics``. @@ -516,7 +532,9 @@ Returns the absolute URL for the given path, including the protocol and host. Fo .. code-block:: python - absolute_url = datasette.absolute_url(request, "/dbname/table.json") + absolute_url = datasette.absolute_url( + request, "/dbname/table.json" + ) # Would return "http://localhost:8001/dbname/table.json" The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`setting_force_https_urls` configuration setting is taken into account. @@ -578,7 +596,9 @@ These methods can be used with :ref:`internals_datasette_urls` - for example: table_json = ( await datasette.client.get( - datasette.urls.table("fixtures", "facetable", format="json") + datasette.urls.table( + "fixtures", "facetable", format="json" + ) ) ).json() @@ -754,6 +774,7 @@ Example usage: "select sqlite_version()" ).fetchall()[0][0] + version = await db.execute_fn(get_version) .. _database_execute_write: @@ -789,7 +810,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() <https://d await db.execute_write_many( "insert into characters (id, name) values (?, ?)", - [(1, "Melanie"), (2, "Selma"), (2, "Viktor")] + [(1, "Melanie"), (2, "Selma"), (2, "Viktor")], ) .. _database_execute_write_fn: @@ -811,10 +832,15 @@ For example: def delete_and_return_count(conn): conn.execute("delete from some_table where id > 5") - return conn.execute("select count(*) from some_table").fetchone()[0] + return conn.execute( + "select count(*) from some_table" + ).fetchone()[0] + try: - num_rows_left = await database.execute_write_fn(delete_and_return_count) + num_rows_left = await database.execute_write_fn( + delete_and_return_count + ) except Exception as e: print("An error occurred:", e) @@ -1021,6 +1047,7 @@ This example uses trace to record the start, end and duration of any HTTP GET re from datasette.tracer import trace import httpx + async def fetch_url(url): with trace("fetch-url", url=url): async with httpx.AsyncClient() as client: @@ -1051,9 +1078,9 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h from datasette import hookimpl from datasette import tracer + @hookimpl def register_routes(): - async def parallel_queries(datasette): db = datasette.get_database() with tracer.trace_child_tasks(): @@ -1061,7 +1088,12 @@ This example uses the :ref:`register_routes() <plugin_register_routes>` plugin h db.execute("select 1"), db.execute("select 2"), ) - return Response.json({"one": one.single_value(), "two": two.single_value()}) + return Response.json( + { + "one": one.single_value(), + "two": two.single_value(), + } + ) return [ (r"/parallel-queries$", parallel_queries), diff --git a/docs/json_api.rst b/docs/json_api.rst index aa6fcdaa..d3fdb1e4 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -446,7 +446,7 @@ Most of the HTML pages served by Datasette provide a mechanism for discovering t You can find this near the top of the source code of those pages, looking like this: -.. code-block:: python +.. code-block:: html <link rel="alternate" type="application/json+datasette" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 67842fc4..ace206b7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -44,9 +44,12 @@ aggregates and collations. For example: from datasette import hookimpl import random + @hookimpl def prepare_connection(conn): - conn.create_function('random_integer', 2, random.randint) + conn.create_function( + "random_integer", 2, random.randint + ) This registers a SQL function called ``random_integer`` which takes two arguments and can be called like this:: @@ -72,9 +75,10 @@ example: from datasette import hookimpl + @hookimpl def prepare_jinja2_environment(env): - env.filters['uppercase'] = lambda u: u.upper() + env.filters["uppercase"] = lambda u: u.upper() You can now use this filter in your custom templates like so:: @@ -127,9 +131,7 @@ Here's an example plugin that adds a ``"user_agent"`` variable to the template c @hookimpl def extra_template_vars(request): - return { - "user_agent": request.headers.get("user-agent") - } + return {"user_agent": request.headers.get("user-agent")} This example returns an awaitable function which adds a list of ``hidden_table_names`` to the context: @@ -140,9 +142,12 @@ This example returns an awaitable function which adds a list of ``hidden_table_n async def hidden_table_names(): if database: db = datasette.databases[database] - return {"hidden_table_names": await db.hidden_table_names()} + return { + "hidden_table_names": await db.hidden_table_names() + } else: return {} + return hidden_table_names And here's an example which adds a ``sql_first(sql_query)`` function which executes a SQL statement and returns the first column of the first row of results: @@ -152,8 +157,15 @@ And here's an example which adds a ``sql_first(sql_query)`` function which execu @hookimpl def extra_template_vars(datasette, database): async def sql_first(sql, dbname=None): - dbname = dbname or database or next(iter(datasette.databases.keys())) - return (await datasette.execute(dbname, sql)).rows[0][0] + dbname = ( + dbname + or database + or next(iter(datasette.databases.keys())) + ) + return (await datasette.execute(dbname, sql)).rows[ + 0 + ][0] + return {"sql_first": sql_first} You can then use the new function in a template like so:: @@ -178,6 +190,7 @@ This can be a list of URLs: from datasette import hookimpl + @hookimpl def extra_css_urls(): return [ @@ -191,10 +204,12 @@ Or a list of dictionaries defining both a URL and an @hookimpl def extra_css_urls(): - return [{ - "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css", - "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4", - }] + return [ + { + "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css", + "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4", + } + ] This function can also return an awaitable function, useful if it needs to run any async code: @@ -204,7 +219,9 @@ This function can also return an awaitable function, useful if it needs to run a def extra_css_urls(datasette): async def inner(): db = datasette.get_database() - results = await db.execute("select url from css_files") + results = await db.execute( + "select url from css_files" + ) return [r[0] for r in results] return inner @@ -225,12 +242,15 @@ return a list of URLs, a list of dictionaries or an awaitable function that retu from datasette import hookimpl + @hookimpl def extra_js_urls(): - return [{ - "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js", - "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo", - }] + return [ + { + "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js", + "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo", + } + ] You can also return URLs to files from your plugin's ``static/`` directory, if you have one: @@ -239,9 +259,7 @@ you have one: @hookimpl def extra_js_urls(): - return [ - "/-/static-plugins/your-plugin/app.js" - ] + return ["/-/static-plugins/your-plugin/app.js"] Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. @@ -251,9 +269,11 @@ If your code uses `JavaScript modules <https://developer.mozilla.org/en-US/docs/ @hookimpl def extra_js_urls(): - return [{ - "url": "/-/static-plugins/your-plugin/app.js", - "module": True + return [ + { + "url": "/-/static-plugins/your-plugin/app.js", + "module": True, + } ] Examples: `datasette-cluster-map <https://datasette.io/plugins/datasette-cluster-map>`_, `datasette-vega <https://datasette.io/plugins/datasette-vega>`_ @@ -281,7 +301,7 @@ Use a dictionary if you want to specify that the code should be placed in a ``<s def extra_body_script(): return { "module": True, - "script": "console.log('Your JavaScript goes here...')" + "script": "console.log('Your JavaScript goes here...')", } This will add the following to the end of your page: @@ -311,7 +331,9 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ .. code-block:: python from datasette import hookimpl - from datasette.publish.common import add_common_publish_arguments_and_options + from datasette.publish.common import ( + add_common_publish_arguments_and_options, + ) import click @@ -345,7 +367,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ about_url, api_key, ): - # Your implementation goes here + ... Examples: `datasette-publish-fly <https://datasette.io/plugins/datasette-publish-fly>`_, `datasette-publish-vercel <https://datasette.io/plugins/datasette-publish-vercel>`_ @@ -400,7 +422,9 @@ If the value matches that pattern, the plugin returns an HTML link element: if not isinstance(value, str): return None stripped = value.strip() - if not stripped.startswith("{") and stripped.endswith("}"): + if not stripped.startswith("{") and stripped.endswith( + "}" + ): return None try: data = json.loads(value) @@ -412,14 +436,18 @@ If the value matches that pattern, the plugin returns an HTML link element: return None href = data["href"] if not ( - href.startswith("/") or href.startswith("http://") + href.startswith("/") + or href.startswith("http://") or href.startswith("https://") ): return None - return markupsafe.Markup('<a href="{href}">{label}</a>'.format( - href=markupsafe.escape(data["href"]), - label=markupsafe.escape(data["label"] or "") or " " - )) + return markupsafe.Markup( + '<a href="{href}">{label}</a>'.format( + href=markupsafe.escape(data["href"]), + label=markupsafe.escape(data["label"] or "") + or " ", + ) + ) Examples: `datasette-render-binary <https://datasette.io/plugins/datasette-render-binary>`_, `datasette-render-markdown <https://datasette.io/plugins/datasette-render-markdown>`__, `datasette-json-html <https://datasette.io/plugins/datasette-json-html>`__ @@ -516,7 +544,7 @@ Here is a more complex example: return Response( "\n".join(lines), content_type="text/plain; charset=utf-8", - headers={"x-sqlite-version": result.first()[0]} + headers={"x-sqlite-version": result.first()[0]}, ) And here is an example ``can_render`` function which returns ``True`` only if the query results contain the columns ``atom_id``, ``atom_title`` and ``atom_updated``: @@ -524,7 +552,11 @@ And here is an example ``can_render`` function which returns ``True`` only if th .. code-block:: python def can_render_demo(columns): - return {"atom_id", "atom_title", "atom_updated"}.issubset(columns) + return { + "atom_id", + "atom_title", + "atom_updated", + }.issubset(columns) Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__ @@ -548,16 +580,14 @@ Return a list of ``(regex, view_function)`` pairs, something like this: async def hello_from(request): name = request.url_vars["name"] - return Response.html("Hello from {}".format( - html.escape(name) - )) + return Response.html( + "Hello from {}".format(html.escape(name)) + ) @hookimpl def register_routes(): - return [ - (r"^/hello-from/(?P<name>.*)$", hello_from) - ] + return [(r"^/hello-from/(?P<name>.*)$", hello_from)] The view functions can take a number of different optional arguments. The corresponding argument will be passed to your function depending on its named parameters - a form of dependency injection. @@ -606,10 +636,13 @@ This example registers a new ``datasette verify file1.db file2.db`` command that import click import sqlite3 + @hookimpl def register_commands(cli): @cli.command() - @click.argument("files", type=click.Path(exists=True), nargs=-1) + @click.argument( + "files", type=click.Path(exists=True), nargs=-1 + ) def verify(files): "Verify that files can be opened by Datasette" for file in files: @@ -617,7 +650,9 @@ This example registers a new ``datasette verify file1.db file2.db`` command that try: conn.execute("select * from sqlite_master") except sqlite3.DatabaseError: - raise click.ClickException("Invalid database: {}".format(file)) + raise click.ClickException( + "Invalid database: {}".format(file) + ) The new command can then be executed like so:: @@ -656,15 +691,18 @@ Each Facet subclass implements a new type of facet operation. The class should l async def suggest(self): # Use self.sql and self.params to suggest some facets suggested_facets = [] - suggested_facets.append({ - "name": column, # Or other unique name - # Construct the URL that will enable this facet: - "toggle_url": self.ds.absolute_url( - self.request, path_with_added_args( - self.request, {"_facet": column} - ) - ), - }) + suggested_facets.append( + { + "name": column, # Or other unique name + # Construct the URL that will enable this facet: + "toggle_url": self.ds.absolute_url( + self.request, + path_with_added_args( + self.request, {"_facet": column} + ), + ), + } + ) return suggested_facets async def facet_results(self): @@ -678,18 +716,25 @@ Each Facet subclass implements a new type of facet operation. The class should l try: facet_results_values = [] # More calculations... - facet_results_values.append({ - "value": value, - "label": label, - "count": count, - "toggle_url": self.ds.absolute_url(self.request, toggle_path), - "selected": selected, - }) - facet_results.append({ - "name": column, - "results": facet_results_values, - "truncated": len(facet_rows_results) > facet_size, - }) + facet_results_values.append( + { + "value": value, + "label": label, + "count": count, + "toggle_url": self.ds.absolute_url( + self.request, toggle_path + ), + "selected": selected, + } + ) + facet_results.append( + { + "name": column, + "results": facet_results_values, + "truncated": len(facet_rows_results) + > facet_size, + } + ) except QueryInterrupted: facets_timed_out.append(column) @@ -728,21 +773,33 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att def asgi_wrapper(datasette): def wrap_with_databases_header(app): @wraps(app) - async def add_x_databases_header(scope, receive, send): + async def add_x_databases_header( + scope, receive, send + ): async def wrapped_send(event): if event["type"] == "http.response.start": - original_headers = event.get("headers") or [] + original_headers = ( + event.get("headers") or [] + ) event = { "type": event["type"], "status": event["status"], - "headers": original_headers + [ - [b"x-databases", - ", ".join(datasette.databases.keys()).encode("utf-8")] + "headers": original_headers + + [ + [ + b"x-databases", + ", ".join( + datasette.databases.keys() + ).encode("utf-8"), + ] ], } await send(event) + await app(scope, receive, wrapped_send) + return add_x_databases_header + return wrap_with_databases_header Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__ @@ -759,7 +816,9 @@ This hook fires when the Datasette application server first starts up. You can i @hookimpl def startup(datasette): config = datasette.plugin_config("my-plugin") or {} - assert "required-setting" in config, "my-plugin requires setting required-setting" + assert ( + "required-setting" in config + ), "my-plugin requires setting required-setting" Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries: @@ -770,9 +829,12 @@ Or you can return an async function which will be awaited on startup. Use this o async def inner(): db = datasette.get_database() if "my_table" not in await db.table_names(): - await db.execute_write(""" + await db.execute_write( + """ create table my_table (mycol text) - """) + """ + ) + return inner Potential use-cases: @@ -815,6 +877,7 @@ Ues this hook to return a dictionary of additional :ref:`canned query <canned_qu from datasette import hookimpl + @hookimpl def canned_queries(datasette, database): if database == "mydb": @@ -830,15 +893,20 @@ The hook can alternatively return an awaitable function that returns a list. Her from datasette import hookimpl + @hookimpl def canned_queries(datasette, database): async def inner(): db = datasette.get_database(database) if await db.table_exists("saved_queries"): - results = await db.execute("select name, sql from saved_queries") - return {result["name"]: { - "sql": result["sql"] - } for result in results} + results = await db.execute( + "select name, sql from saved_queries" + ) + return { + result["name"]: {"sql": result["sql"]} + for result in results + } + return inner The actor parameter can be used to include the currently authenticated actor in your decision. Here's an example that returns saved queries that were saved by that actor: @@ -847,19 +915,23 @@ The actor parameter can be used to include the currently authenticated actor in from datasette import hookimpl + @hookimpl def canned_queries(datasette, database, actor): async def inner(): db = datasette.get_database(database) - if actor is not None and await db.table_exists("saved_queries"): + if actor is not None and await db.table_exists( + "saved_queries" + ): results = await db.execute( - "select name, sql from saved_queries where actor_id = :id", { - "id": actor["id"] - } + "select name, sql from saved_queries where actor_id = :id", + {"id": actor["id"]}, ) - return {result["name"]: { - "sql": result["sql"] - } for result in results} + return { + result["name"]: {"sql": result["sql"]} + for result in results + } + return inner Example: `datasette-saved-queries <https://datasette.io/plugins/datasette-saved-queries>`__ @@ -888,9 +960,12 @@ Here's an example that authenticates the actor based on an incoming API key: SECRET_KEY = "this-is-a-secret" + @hookimpl def actor_from_request(datasette, request): - authorization = request.headers.get("authorization") or "" + authorization = ( + request.headers.get("authorization") or "" + ) expected = "Bearer {}".format(SECRET_KEY) if secrets.compare_digest(authorization, expected): @@ -906,6 +981,7 @@ Instead of returning a dictionary, this function can return an awaitable functio from datasette import hookimpl + @hookimpl def actor_from_request(datasette, request): async def inner(): @@ -914,7 +990,8 @@ Instead of returning a dictionary, this function can return an awaitable functio return None # Look up ?_token=xxx in sessions table result = await datasette.get_database().execute( - "select count(*) from sessions where token = ?", [token] + "select count(*) from sessions where token = ?", + [token], ) if result.first()[0]: return {"token": token} @@ -952,7 +1029,7 @@ The hook should return an instance of ``datasette.filters.FilterArguments`` whic where_clauses=["id > :max_id"], params={"max_id": 5}, human_descriptions=["max_id is greater than 5"], - extra_context={} + extra_context={}, ) The arguments to the ``FilterArguments`` class constructor are as follows: @@ -973,10 +1050,13 @@ This example plugin causes 0 results to be returned if ``?_nothing=1`` is added from datasette import hookimpl from datasette.filters import FilterArguments + @hookimpl def filters_from_request(self, request): if request.args.get("_nothing"): - return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) + return FilterArguments( + ["1 = 0"], human_descriptions=["NOTHING"] + ) Example: `datasette-leaflet-freedraw <https://datasette.io/plugins/datasette-leaflet-freedraw>`_ @@ -1006,6 +1086,7 @@ Here's an example plugin which randomly selects if a permission should be allowe from datasette import hookimpl import random + @hookimpl def permission_allowed(action): if action != "view-instance": @@ -1024,11 +1105,16 @@ Here's an example that allows users to view the ``admin_log`` table only if thei async def inner(): if action == "execute-sql" and resource == "staff": return False - if action == "view-table" and resource == ("staff", "admin_log"): + if action == "view-table" and resource == ( + "staff", + "admin_log", + ): if not actor: return False user_id = actor["id"] - return await datasette.get_database("staff").execute( + return await datasette.get_database( + "staff" + ).execute( "select count(*) from admin_users where user_id = :user_id", {"user_id": user_id}, ) @@ -1059,18 +1145,21 @@ This example registers two new magic parameters: ``:_request_http_version`` retu from uuid import uuid4 + def uuid(key, request): if key == "new": return str(uuid4()) else: raise KeyError + def request(key, request): if key == "http_version": return request.scope["http_version"] else: raise KeyError + @hookimpl def register_magic_parameters(datasette): return [ @@ -1103,9 +1192,12 @@ This example returns a redirect to a ``/-/login`` page: from datasette import hookimpl from urllib.parse import urlencode + @hookimpl def forbidden(request, message): - return Response.redirect("/-/login?=" + urlencode({"message": message})) + return Response.redirect( + "/-/login?=" + urlencode({"message": message}) + ) The function can alternatively return an awaitable function if it needs to make any asynchronous method calls. This example renders a template: @@ -1114,10 +1206,15 @@ The function can alternatively return an awaitable function if it needs to make from datasette import hookimpl from datasette.utils.asgi import Response + @hookimpl def forbidden(datasette): async def inner(): - return Response.html(await datasette.render_template("forbidden.html")) + return Response.html( + await datasette.render_template( + "forbidden.html" + ) + ) return inner @@ -1147,11 +1244,17 @@ This example adds a new menu item but only if the signed in user is ``"root"``: from datasette import hookimpl + @hookimpl def menu_links(datasette, actor): if actor and actor.get("id") == "root": return [ - {"href": datasette.urls.path("/-/edit-schema"), "label": "Edit schema"}, + { + "href": datasette.urls.path( + "/-/edit-schema" + ), + "label": "Edit schema", + }, ] Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. @@ -1188,13 +1291,20 @@ This example adds a new table action if the signed in user is ``"root"``: from datasette import hookimpl + @hookimpl def table_actions(datasette, actor): if actor and actor.get("id") == "root": - return [{ - "href": datasette.urls.path("/-/edit-schema/{}/{}".format(database, table)), - "label": "Edit schema for this table", - }] + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/{}".format( + database, table + ) + ), + "label": "Edit schema for this table", + } + ] Example: `datasette-graphql <https://datasette.io/plugins/datasette-graphql>`_ @@ -1238,6 +1348,7 @@ This example will disable CSRF protection for that specific URL path: from datasette import hookimpl + @hookimpl def skip_csrf(scope): return scope["path"] == "/submit-comment" @@ -1278,7 +1389,9 @@ This hook is responsible for returning a dictionary corresponding to Datasette : "description": get_instance_description(datasette), "databases": [], } - for db_name, db_data_dict in get_my_database_meta(datasette, database, table, key): + for db_name, db_data_dict in get_my_database_meta( + datasette, database, table, key + ): metadata["databases"][db_name] = db_data_dict # whatever we return here will be merged with any other plugins using this hook and # will be overwritten by a local metadata.yaml if one exists! From 498e1536f5f3e69c50934c0c031055e0af770bf6 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 09:08:56 -0700 Subject: [PATCH 0688/1866] One more blacken-docs test, refs #1718 --- docs/testing_plugins.rst | 45 ++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 8e4e3f91..6361d744 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -19,7 +19,10 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl response = await datasette.client.get("/-/plugins.json") assert response.status_code == 200 installed_plugins = {p["name"] for p in response.json()} - assert "datasette-plugin-template-demo" in installed_plugins + assert ( + "datasette-plugin-template-demo" + in installed_plugins + ) This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. @@ -37,9 +40,7 @@ If you are building an installable package you can add them as test dependencies setup( name="datasette-my-plugin", # ... - extras_require={ - "test": ["pytest", "pytest-asyncio"] - }, + extras_require={"test": ["pytest", "pytest-asyncio"]}, tests_require=["datasette-my-plugin[test]"], ) @@ -87,31 +88,34 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data import pytest import sqlite_utils + @pytest.fixture(scope="session") def datasette(tmp_path_factory): db_directory = tmp_path_factory.mktemp("dbs") db_path = db_directory / "test.db" db = sqlite_utils.Database(db_path) - db["dogs"].insert_all([ - {"id": 1, "name": "Cleo", "age": 5}, - {"id": 2, "name": "Pancakes", "age": 4} - ], pk="id") + db["dogs"].insert_all( + [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Pancakes", "age": 4}, + ], + pk="id", + ) datasette = Datasette( [db_path], metadata={ "databases": { "test": { "tables": { - "dogs": { - "title": "Some dogs" - } + "dogs": {"title": "Some dogs"} } } } - } + }, ) return datasette + @pytest.mark.asyncio async def test_example_table_json(datasette): response = await datasette.client.get("/test/dogs.json?_shape=array") @@ -121,6 +125,7 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data {"id": 2, "name": "Pancakes", "age": 4}, ] + @pytest.mark.asyncio async def test_example_table_html(datasette): response = await datasette.client.get("/test/dogs") @@ -137,6 +142,7 @@ If you want to create that test database repeatedly for every individual test fu @pytest.fixture def datasette(tmp_path_factory): # This fixture will be executed repeatedly for every test + ... .. _testing_plugins_pytest_httpx: @@ -197,14 +203,17 @@ Here's a test for that plugin that mocks the HTTPX outbound request: async def test_outbound_http_call(httpx_mock): httpx_mock.add_response( - url='https://www.example.com/', - text='Hello world', + url="https://www.example.com/", + text="Hello world", ) datasette = Datasette([], memory=True) - response = await datasette.client.post("/-/fetch-url", data={ - "url": "https://www.example.com/" - }) + response = await datasette.client.post( + "/-/fetch-url", + data={"url": "https://www.example.com/"}, + ) assert response.text == "Hello world" outbound_request = httpx_mock.get_request() - assert outbound_request.url == "https://www.example.com/" + assert ( + outbound_request.url == "https://www.example.com/" + ) From 289e4cf80a14f05f791b218f092556148b49a0fa Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 09:17:59 -0700 Subject: [PATCH 0689/1866] Finished applying blacken-docs, closes #1718 --- .github/workflows/test.yml | 3 +-- docs/testing_plugins.rst | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38b62995..8d916e49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,5 @@ jobs: cog --check docs/*.rst - name: Check if blacken-docs needs to be run run: | + # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst - # This fails if a diff was generated: - git diff-index --quiet HEAD -- diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 6361d744..1bbaaac1 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -118,7 +118,9 @@ Here's an example that uses the `sqlite-utils library <https://sqlite-utils.data @pytest.mark.asyncio async def test_example_table_json(datasette): - response = await datasette.client.get("/test/dogs.json?_shape=array") + response = await datasette.client.get( + "/test/dogs.json?_shape=array" + ) assert response.status_code == 200 assert response.json() == [ {"id": 1, "name": "Cleo", "age": 5}, From 7463b051cf8d7f856df5eba9f7aa944183ebabe5 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 Apr 2022 09:59:20 -0700 Subject: [PATCH 0690/1866] Cosmetic tweaks after blacken-docs, refs #1718 --- docs/plugin_hooks.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ace206b7..4560ec9a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -162,9 +162,8 @@ And here's an example which adds a ``sql_first(sql_query)`` function which execu or database or next(iter(datasette.databases.keys())) ) - return (await datasette.execute(dbname, sql)).rows[ - 0 - ][0] + result = await datasette.execute(dbname, sql) + return result.rows[0][0] return {"sql_first": sql_first} @@ -422,8 +421,8 @@ If the value matches that pattern, the plugin returns an HTML link element: if not isinstance(value, str): return None stripped = value.strip() - if not stripped.startswith("{") and stripped.endswith( - "}" + if not ( + stripped.startswith("{") and stripped.endswith("}") ): return None try: From 579f59dcec43a91dd7d404e00b87a00afd8515f2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 25 Apr 2022 11:33:35 -0700 Subject: [PATCH 0691/1866] Refactor to remove RowTableShared class, closes #1719 Refs #1715 --- datasette/app.py | 3 +- datasette/views/row.py | 142 +++++++++++ datasette/views/table.py | 497 +++++++++++++++------------------------ 3 files changed, 328 insertions(+), 314 deletions(-) create mode 100644 datasette/views/row.py diff --git a/datasette/app.py b/datasette/app.py index c9eede26..d269372c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -40,7 +40,8 @@ from .views.special import ( PermissionsDebugView, MessagesDebugView, ) -from .views.table import RowView, TableView +from .views.table import TableView +from .views.row import RowView from .renderer import json_renderer from .url_builder import Urls from .database import Database, QueryInterrupted diff --git a/datasette/views/row.py b/datasette/views/row.py new file mode 100644 index 00000000..b1c7362d --- /dev/null +++ b/datasette/views/row.py @@ -0,0 +1,142 @@ +from datasette.utils.asgi import NotFound +from datasette.database import QueryInterrupted +from .base import DataView +from datasette.utils import ( + tilde_decode, + urlsafe_components, + to_css_class, + escape_sqlite, +) +from .table import _sql_params_pks, display_columns_and_rows + + +class RowView(DataView): + name = "row" + + async def data(self, request, default_labels=False): + database_route = tilde_decode(request.url_vars["database"]) + table = tilde_decode(request.url_vars["table"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + await self.ds.ensure_permissions( + request.actor, + [ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + pk_values = urlsafe_components(request.url_vars["pks"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + sql, params, pks = await _sql_params_pks(db, table, pk_values) + results = await db.execute(sql, params, truncate=True) + columns = [r[0] for r in results.description] + rows = list(results.rows) + if not rows: + raise NotFound(f"Record not found: {pk_values}") + + async def template_data(): + display_columns, display_rows = await display_columns_and_rows( + self.ds, + database, + table, + results.description, + rows, + link_column=False, + truncate_cells=0, + ) + for column in display_columns: + column["sortable"] = False + return { + "foreign_key_tables": await self.foreign_key_tables( + database, table, pk_values + ), + "display_columns": display_columns, + "display_rows": display_rows, + "custom_table_templates": [ + f"_table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", + "_table.html", + ], + "metadata": (self.ds.metadata("databases") or {}) + .get(database, {}) + .get("tables", {}) + .get(table, {}), + } + + data = { + "database": database, + "table": table, + "rows": rows, + "columns": columns, + "primary_keys": pks, + "primary_key_values": pk_values, + "units": self.ds.table_metadata(database, table).get("units", {}), + } + + if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): + data["foreign_key_tables"] = await self.foreign_key_tables( + database, table, pk_values + ) + + return ( + data, + template_data, + ( + f"row-{to_css_class(database)}-{to_css_class(table)}.html", + "row.html", + ), + ) + + async def foreign_key_tables(self, database, table, pk_values): + if len(pk_values) != 1: + return [] + db = self.ds.databases[database] + all_foreign_keys = await db.get_all_foreign_keys() + foreign_keys = all_foreign_keys[table]["incoming"] + if len(foreign_keys) == 0: + return [] + + sql = "select " + ", ".join( + [ + "(select count(*) from {table} where {column}=:id)".format( + table=escape_sqlite(fk["other_table"]), + column=escape_sqlite(fk["other_column"]), + ) + for fk in foreign_keys + ] + ) + try: + rows = list(await db.execute(sql, {"id": pk_values[0]})) + except QueryInterrupted: + # Almost certainly hit the timeout + return [] + + foreign_table_counts = dict( + zip( + [(fk["other_table"], fk["other_column"]) for fk in foreign_keys], + list(rows[0]), + ) + ) + foreign_key_tables = [] + for fk in foreign_keys: + count = ( + foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0 + ) + key = fk["other_column"] + if key.startswith("_"): + key += "__exact" + link = "{}?{}={}".format( + self.ds.urls.table(database, fk["other_table"]), + key, + ",".join(pk_values), + ) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) + return foreign_key_tables diff --git a/datasette/views/table.py b/datasette/views/table.py index dc85165e..37fb2ebb 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,4 +1,3 @@ -import urllib import itertools import json @@ -9,7 +8,6 @@ from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, CustomRow, - MultiParams, append_querystring, compound_keys_after_sql, format_bytes, @@ -21,7 +19,6 @@ from datasette.utils import ( is_url, path_from_row_pks, path_with_added_args, - path_with_format, path_with_removed_args, path_with_replaced_args, to_css_class, @@ -68,7 +65,9 @@ class Row: return json.dumps(d, default=repr, indent=2) -class RowTableShared(DataView): +class TableView(DataView): + name = "table" + async def sortable_columns_for_table(self, database, table, use_rowid): db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) @@ -89,193 +88,6 @@ class RowTableShared(DataView): expandables.append((fk, label_column)) return expandables - async def display_columns_and_rows( - self, database, table, description, rows, link_column=False, truncate_cells=0 - ): - """Returns columns, rows for specified table - including fancy foreign key treatment""" - db = self.ds.databases[database] - table_metadata = self.ds.table_metadata(database, table) - column_descriptions = table_metadata.get("columns") or {} - column_details = {col.name: col for col in await db.table_column_details(table)} - sortable_columns = await self.sortable_columns_for_table(database, table, True) - pks = await db.primary_keys(table) - pks_for_display = pks - if not pks_for_display: - pks_for_display = ["rowid"] - - columns = [] - for r in description: - if r[0] == "rowid" and "rowid" not in column_details: - type_ = "integer" - notnull = 0 - else: - type_ = column_details[r[0]].type - notnull = column_details[r[0]].notnull - columns.append( - { - "name": r[0], - "sortable": r[0] in sortable_columns, - "is_pk": r[0] in pks_for_display, - "type": type_, - "notnull": notnull, - "description": column_descriptions.get(r[0]), - } - ) - - column_to_foreign_key_table = { - fk["column"]: fk["other_table"] - for fk in await db.foreign_keys_for_table(table) - } - - cell_rows = [] - base_url = self.ds.setting("base_url") - for row in rows: - cells = [] - # Unless we are a view, the first column is a link - either to the rowid - # or to the simple or compound primary key - if link_column: - is_special_link_column = len(pks) != 1 - pk_path = path_from_row_pks(row, pks, not pks, False) - cells.append( - { - "column": pks[0] if len(pks) == 1 else "Link", - "value_type": "pk", - "is_special_link_column": is_special_link_column, - "raw": pk_path, - "value": markupsafe.Markup( - '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format( - base_url=base_url, - table_path=self.ds.urls.table(database, table), - flat_pks=str(markupsafe.escape(pk_path)), - flat_pks_quoted=path_from_row_pks(row, pks, not pks), - ) - ), - } - ) - - for value, column_dict in zip(row, columns): - column = column_dict["name"] - if link_column and len(pks) == 1 and column == pks[0]: - # If there's a simple primary key, don't repeat the value as it's - # already shown in the link column. - continue - - # First let the plugins have a go - # pylint: disable=no-member - plugin_display_value = None - for candidate in pm.hook.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - display_value = plugin_display_value - elif isinstance(value, bytes): - formatted = format_bytes(len(value)) - display_value = markupsafe.Markup( - '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format( - self.ds.urls.row_blob( - database, - table, - path_from_row_pks(row, pks, not pks), - column, - ), - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", - len(value), - "" if len(value) == 1 else "s", - ) - ) - elif isinstance(value, dict): - # It's an expanded foreign key - display link to other row - label = value["label"] - value = value["value"] - # The table we link to depends on the column - other_table = column_to_foreign_key_table[column] - link_template = ( - LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE - ) - display_value = markupsafe.Markup( - link_template.format( - database=database, - base_url=base_url, - table=tilde_encode(other_table), - link_id=tilde_encode(str(value)), - id=str(markupsafe.escape(value)), - label=str(markupsafe.escape(label)) or "-", - ) - ) - elif value in ("", None): - display_value = markupsafe.Markup(" ") - elif is_url(str(value).strip()): - display_value = markupsafe.Markup( - '<a href="{url}">{url}</a>'.format( - url=markupsafe.escape(value.strip()) - ) - ) - elif column in table_metadata.get("units", {}) and value != "": - # Interpret units using pint - value = value * ureg(table_metadata["units"][column]) - # Pint uses floating point which sometimes introduces errors in the compact - # representation, which we have to round off to avoid ugliness. In the vast - # majority of cases this rounding will be inconsequential. I hope. - value = round(value.to_compact(), 6) - display_value = markupsafe.Markup( - f"{value:~P}".replace(" ", " ") - ) - else: - display_value = str(value) - if truncate_cells and len(display_value) > truncate_cells: - display_value = display_value[:truncate_cells] + "\u2026" - - cells.append( - { - "column": column, - "value": display_value, - "raw": value, - "value_type": "none" - if value is None - else str(type(value).__name__), - } - ) - cell_rows.append(Row(cells)) - - if link_column: - # Add the link column header. - # If it's a simple primary key, we have to remove and re-add that column name at - # the beginning of the header row. - first_column = None - if len(pks) == 1: - columns = [col for col in columns if col["name"] != pks[0]] - first_column = { - "name": pks[0], - "sortable": len(pks) == 1, - "is_pk": True, - "type": column_details[pks[0]].type, - "notnull": column_details[pks[0]].notnull, - } - else: - first_column = { - "name": "Link", - "sortable": False, - "is_pk": False, - "type": "", - "notnull": 0, - } - columns = [first_column] + columns - return columns, cell_rows - - -class TableView(RowTableShared): - name = "table" - async def post(self, request): database_route = tilde_decode(request.url_vars["database"]) try: @@ -807,13 +619,17 @@ class TableView(RowTableShared): async def extra_template(): nonlocal sort - display_columns, display_rows = await self.display_columns_and_rows( + display_columns, display_rows = await display_columns_and_rows( + self.ds, database, table, results.description, rows, link_column=not is_view, truncate_cells=self.ds.setting("truncate_cells_html"), + sortable_columns=await self.sortable_columns_for_table( + database, table, use_rowid=True + ), ) metadata = ( (self.ds.metadata("databases") or {}) @@ -948,132 +764,187 @@ async def _sql_params_pks(db, table, pk_values): return sql, params, pks -class RowView(RowTableShared): - name = "row" +async def display_columns_and_rows( + datasette, + database, + table, + description, + rows, + link_column=False, + truncate_cells=0, + sortable_columns=None, +): + """Returns columns, rows for specified table - including fancy foreign key treatment""" + sortable_columns = sortable_columns or set() + db = datasette.databases[database] + table_metadata = datasette.table_metadata(database, table) + column_descriptions = table_metadata.get("columns") or {} + column_details = {col.name: col for col in await db.table_column_details(table)} + pks = await db.primary_keys(table) + pks_for_display = pks + if not pks_for_display: + pks_for_display = ["rowid"] - async def data(self, request, default_labels=False): - database_route = tilde_decode(request.url_vars["database"]) - table = tilde_decode(request.url_vars["table"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - await self.ds.ensure_permissions( - request.actor, - [ - ("view-table", (database, table)), - ("view-database", database), - "view-instance", - ], - ) - pk_values = urlsafe_components(request.url_vars["pks"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - sql, params, pks = await _sql_params_pks(db, table, pk_values) - results = await db.execute(sql, params, truncate=True) - columns = [r[0] for r in results.description] - rows = list(results.rows) - if not rows: - raise NotFound(f"Record not found: {pk_values}") - - async def template_data(): - display_columns, display_rows = await self.display_columns_and_rows( - database, - table, - results.description, - rows, - link_column=False, - truncate_cells=0, - ) - for column in display_columns: - column["sortable"] = False - return { - "foreign_key_tables": await self.foreign_key_tables( - database, table, pk_values - ), - "display_columns": display_columns, - "display_rows": display_rows, - "custom_table_templates": [ - f"_table-{to_css_class(database)}-{to_css_class(table)}.html", - f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", - "_table.html", - ], - "metadata": (self.ds.metadata("databases") or {}) - .get(database, {}) - .get("tables", {}) - .get(table, {}), + columns = [] + for r in description: + if r[0] == "rowid" and "rowid" not in column_details: + type_ = "integer" + notnull = 0 + else: + type_ = column_details[r[0]].type + notnull = column_details[r[0]].notnull + columns.append( + { + "name": r[0], + "sortable": r[0] in sortable_columns, + "is_pk": r[0] in pks_for_display, + "type": type_, + "notnull": notnull, + "description": column_descriptions.get(r[0]), } - - data = { - "database": database, - "table": table, - "rows": rows, - "columns": columns, - "primary_keys": pks, - "primary_key_values": pk_values, - "units": self.ds.table_metadata(database, table).get("units", {}), - } - - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - return ( - data, - template_data, - ( - f"row-{to_css_class(database)}-{to_css_class(table)}.html", - "row.html", - ), ) - async def foreign_key_tables(self, database, table, pk_values): - if len(pk_values) != 1: - return [] - db = self.ds.databases[database] - all_foreign_keys = await db.get_all_foreign_keys() - foreign_keys = all_foreign_keys[table]["incoming"] - if len(foreign_keys) == 0: - return [] + column_to_foreign_key_table = { + fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table) + } - sql = "select " + ", ".join( - [ - "(select count(*) from {table} where {column}=:id)".format( - table=escape_sqlite(fk["other_table"]), - column=escape_sqlite(fk["other_column"]), + cell_rows = [] + base_url = datasette.setting("base_url") + for row in rows: + cells = [] + # Unless we are a view, the first column is a link - either to the rowid + # or to the simple or compound primary key + if link_column: + is_special_link_column = len(pks) != 1 + pk_path = path_from_row_pks(row, pks, not pks, False) + cells.append( + { + "column": pks[0] if len(pks) == 1 else "Link", + "value_type": "pk", + "is_special_link_column": is_special_link_column, + "raw": pk_path, + "value": markupsafe.Markup( + '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format( + base_url=base_url, + table_path=datasette.urls.table(database, table), + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=path_from_row_pks(row, pks, not pks), + ) + ), + } + ) + + for value, column_dict in zip(row, columns): + column = column_dict["name"] + if link_column and len(pks) == 1 and column == pks[0]: + # If there's a simple primary key, don't repeat the value as it's + # already shown in the link column. + continue + + # First let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + value=value, + column=column, + table=table, + database=database, + datasette=datasette, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + display_value = plugin_display_value + elif isinstance(value, bytes): + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format( + datasette.urls.row_blob( + database, + table, + path_from_row_pks(row, pks, not pks), + column, + ), + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) ) - for fk in foreign_keys - ] - ) - try: - rows = list(await db.execute(sql, {"id": pk_values[0]})) - except QueryInterrupted: - # Almost certainly hit the timeout - return [] + elif isinstance(value, dict): + # It's an expanded foreign key - display link to other row + label = value["label"] + value = value["value"] + # The table we link to depends on the column + other_table = column_to_foreign_key_table[column] + link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE + display_value = markupsafe.Markup( + link_template.format( + database=database, + base_url=base_url, + table=tilde_encode(other_table), + link_id=tilde_encode(str(value)), + id=str(markupsafe.escape(value)), + label=str(markupsafe.escape(label)) or "-", + ) + ) + elif value in ("", None): + display_value = markupsafe.Markup(" ") + elif is_url(str(value).strip()): + display_value = markupsafe.Markup( + '<a href="{url}">{url}</a>'.format( + url=markupsafe.escape(value.strip()) + ) + ) + elif column in table_metadata.get("units", {}) and value != "": + # Interpret units using pint + value = value * ureg(table_metadata["units"][column]) + # Pint uses floating point which sometimes introduces errors in the compact + # representation, which we have to round off to avoid ugliness. In the vast + # majority of cases this rounding will be inconsequential. I hope. + value = round(value.to_compact(), 6) + display_value = markupsafe.Markup(f"{value:~P}".replace(" ", " ")) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" - foreign_table_counts = dict( - zip( - [(fk["other_table"], fk["other_column"]) for fk in foreign_keys], - list(rows[0]), + cells.append( + { + "column": column, + "value": display_value, + "raw": value, + "value_type": "none" + if value is None + else str(type(value).__name__), + } ) - ) - foreign_key_tables = [] - for fk in foreign_keys: - count = ( - foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0 - ) - key = fk["other_column"] - if key.startswith("_"): - key += "__exact" - link = "{}?{}={}".format( - self.ds.urls.table(database, fk["other_table"]), - key, - ",".join(pk_values), - ) - foreign_key_tables.append({**fk, **{"count": count, "link": link}}) - return foreign_key_tables + cell_rows.append(Row(cells)) + + if link_column: + # Add the link column header. + # If it's a simple primary key, we have to remove and re-add that column name at + # the beginning of the header row. + first_column = None + if len(pks) == 1: + columns = [col for col in columns if col["name"] != pks[0]] + first_column = { + "name": pks[0], + "sortable": len(pks) == 1, + "is_pk": True, + "type": column_details[pks[0]].type, + "notnull": column_details[pks[0]].notnull, + } + else: + first_column = { + "name": "Link", + "sortable": False, + "is_pk": False, + "type": "", + "notnull": 0, + } + columns = [first_column] + columns + return columns, cell_rows From c101f0efeec4f6e49298a542c5e2b59236cfa0ff Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 26 Apr 2022 15:34:29 -0700 Subject: [PATCH 0692/1866] datasette-total-page-time example of asgi_wrapper --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4560ec9a..3c9ae2e2 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -801,7 +801,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return wrap_with_databases_header -Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__ +Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `datasette-pyinstrument <https://datasette.io/plugins/datasette-pyinstrument>`__, `datasette-total-page-time <https://datasette.io/plugins/datasette-total-page-time>`__ .. _plugin_hook_startup: From 8a0c38f0b89543e652a968a90d480859cb102510 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 26 Apr 2022 13:56:27 -0700 Subject: [PATCH 0693/1866] Rename database->database_name and table-> table_name, refs #1715 --- datasette/views/table.py | 143 +++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 37fb2ebb..d66adb82 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -68,22 +68,22 @@ class Row: class TableView(DataView): name = "table" - async def sortable_columns_for_table(self, database, table, use_rowid): - db = self.ds.databases[database] - table_metadata = self.ds.table_metadata(database, table) + async def sortable_columns_for_table(self, database_name, table_name, use_rowid): + db = self.ds.databases[database_name] + table_metadata = self.ds.table_metadata(database_name, table_name) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: - sortable_columns = set(await db.table_columns(table)) + sortable_columns = set(await db.table_columns(table_name)) if use_rowid: sortable_columns.add("rowid") return sortable_columns - async def expandable_columns(self, database, table): + async def expandable_columns(self, database_name, table_name): # Returns list of (fk_dict, label_column-or-None) pairs for that table expandables = [] - db = self.ds.databases[database] - for fk in await db.foreign_keys_for_table(table): + db = self.ds.databases[database_name] + for fk in await db.foreign_keys_for_table(table_name): label_column = await db.label_column_for_table(fk["other_table"]) expandables.append((fk, label_column)) return expandables @@ -94,17 +94,19 @@ class TableView(DataView): db = self.ds.get_database(route=database_route) except KeyError: raise NotFound("Database not found: {}".format(database_route)) - database = db.name - table = tilde_decode(request.url_vars["table"]) + database_name = db.name + table_name = tilde_decode(request.url_vars["table"]) # Handle POST to a canned query - canned_query = await self.ds.get_canned_query(database, table, request.actor) + canned_query = await self.ds.get_canned_query( + database_name, table_name, request.actor + ) assert canned_query, "You may only POST to a canned query" return await QueryView(self.ds).data( request, canned_query["sql"], metadata=canned_query, editable=False, - canned_query=table, + canned_query=table_name, named_parameters=canned_query.get("params"), write=bool(canned_query.get("write")), ) @@ -150,45 +152,47 @@ class TableView(DataView): _size=None, ): database_route = tilde_decode(request.url_vars["database"]) - table = tilde_decode(request.url_vars["table"]) + table_name = tilde_decode(request.url_vars["table"]) try: db = self.ds.get_database(route=database_route) except KeyError: raise NotFound("Database not found: {}".format(database_route)) - database = db.name + database_name = db.name # If this is a canned query, not a table, then dispatch to QueryView instead - canned_query = await self.ds.get_canned_query(database, table, request.actor) + canned_query = await self.ds.get_canned_query( + database_name, table_name, request.actor + ) if canned_query: return await QueryView(self.ds).data( request, canned_query["sql"], metadata=canned_query, editable=False, - canned_query=table, + canned_query=table_name, named_parameters=canned_query.get("params"), write=bool(canned_query.get("write")), ) - is_view = bool(await db.get_view_definition(table)) - table_exists = bool(await db.table_exists(table)) + is_view = bool(await db.get_view_definition(table_name)) + table_exists = bool(await db.table_exists(table_name)) # If table or view not found, return 404 if not is_view and not table_exists: - raise NotFound(f"Table not found: {table}") + raise NotFound(f"Table not found: {table_name}") # Ensure user has permission to view this table await self.ds.ensure_permissions( request.actor, [ - ("view-table", (database, table)), - ("view-database", database), + ("view-table", (database_name, table_name)), + ("view-database", database_name), "view-instance", ], ) private = not await self.ds.permission_allowed( - None, "view-table", (database, table), default=True + None, "view-table", (database_name, table_name), default=True ) # Handle ?_filter_column and redirect, if present @@ -216,8 +220,8 @@ class TableView(DataView): ) # Introspect columns and primary keys for table - pks = await db.primary_keys(table) - table_columns = await db.table_columns(table) + pks = await db.primary_keys(table_name) + table_columns = await db.table_columns(table_name) # Take ?_col= and ?_nocol= into account specified_columns = await self.columns_to_select(table_columns, pks, request) @@ -248,7 +252,7 @@ class TableView(DataView): nocount = True nofacet = True - table_metadata = self.ds.table_metadata(database, table) + table_metadata = self.ds.table_metadata(database_name, table_name) units = table_metadata.get("units", {}) # Arguments that start with _ and don't contain a __ are @@ -262,7 +266,7 @@ class TableView(DataView): # Build where clauses from query string arguments filters = Filters(sorted(filter_args), units, ureg) - where_clauses, params = filters.build_where_clauses(table) + where_clauses, params = filters.build_where_clauses(table_name) # Execute filters_from_request plugin hooks - including the default # ones that live in datasette/filters.py @@ -271,8 +275,8 @@ class TableView(DataView): for hook in pm.hook.filters_from_request( request=request, - table=table, - database=database, + table=table_name, + database=database_name, datasette=self.ds, ): filter_arguments = await await_me_maybe(hook) @@ -284,7 +288,7 @@ class TableView(DataView): # Deal with custom sort orders sortable_columns = await self.sortable_columns_for_table( - database, table, use_rowid + database_name, table_name, use_rowid ) sort = request.args.get("_sort") sort_desc = request.args.get("_sort_desc") @@ -309,7 +313,7 @@ class TableView(DataView): order_by = f"{escape_sqlite(sort_desc)} desc" from_sql = "from {table_name} {where}".format( - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=("where {} ".format(" and ".join(where_clauses))) if where_clauses else "", @@ -422,7 +426,7 @@ class TableView(DataView): sql_no_order_no_limit = ( "select {select_all_columns} from {table_name} {where}".format( select_all_columns=select_all_columns, - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=where_clause, ) ) @@ -430,7 +434,7 @@ class TableView(DataView): # This is the SQL that populates the main table on the page sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( select_specified_columns=select_specified_columns, - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=where_clause, order_by=order_by, page_size=page_size + 1, @@ -448,13 +452,13 @@ class TableView(DataView): if ( not db.is_mutable and self.ds.inspect_data - and count_sql == f"select count(*) from {table} " + and count_sql == f"select count(*) from {table_name} " ): # We can use a previously cached table row count try: - filtered_table_rows_count = self.ds.inspect_data[database]["tables"][ - table - ]["count"] + filtered_table_rows_count = self.ds.inspect_data[database_name][ + "tables" + ][table_name]["count"] except KeyError: pass @@ -484,10 +488,10 @@ class TableView(DataView): klass( self.ds, request, - database, + database_name, sql=sql_no_order_no_limit, params=params, - table=table, + table=table_name, metadata=table_metadata, row_count=filtered_table_rows_count, ) @@ -527,7 +531,7 @@ class TableView(DataView): # Expand labeled columns if requested expanded_columns = [] - expandable_columns = await self.expandable_columns(database, table) + expandable_columns = await self.expandable_columns(database_name, table_name) columns_to_expand = None try: all_labels = value_as_boolean(request.args.get("_labels", "")) @@ -554,7 +558,9 @@ class TableView(DataView): values = [row[column_index] for row in rows] # Expand them expanded_labels.update( - await self.ds.expand_foreign_keys(database, table, column, values) + await self.ds.expand_foreign_keys( + database_name, table_name, column, values + ) ) if expanded_labels: # Rewrite the rows @@ -621,21 +627,21 @@ class TableView(DataView): display_columns, display_rows = await display_columns_and_rows( self.ds, - database, - table, + database_name, + table_name, results.description, rows, link_column=not is_view, truncate_cells=self.ds.setting("truncate_cells_html"), sortable_columns=await self.sortable_columns_for_table( - database, table, use_rowid=True + database_name, table_name, use_rowid=True ), ) metadata = ( (self.ds.metadata("databases") or {}) - .get(database, {}) + .get(database_name, {}) .get("tables", {}) - .get(table, {}) + .get(table_name, {}) ) self.ds.update_with_inherited_metadata(metadata) @@ -661,8 +667,8 @@ class TableView(DataView): links = [] for hook in pm.hook.table_actions( datasette=self.ds, - table=table, - database=database, + table=table_name, + database=database_name, actor=request.actor, request=request, ): @@ -703,13 +709,13 @@ class TableView(DataView): "sort_desc": sort_desc, "disable_sort": is_view, "custom_table_templates": [ - f"_table-{to_css_class(database)}-{to_css_class(table)}.html", - f"_table-table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", + f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", "_table.html", ], "metadata": metadata, - "view_definition": await db.get_view_definition(table), - "table_definition": await db.get_table_definition(table), + "view_definition": await db.get_view_definition(table_name), + "table_definition": await db.get_table_definition(table_name), "datasette_allow_facet": "true" if self.ds.setting("allow_facet") else "false", @@ -719,8 +725,8 @@ class TableView(DataView): return ( { - "database": database, - "table": table, + "database": database_name, + "table": table_name, "is_view": is_view, "human_description_en": human_description_en, "rows": rows[:page_size], @@ -738,12 +744,12 @@ class TableView(DataView): "next_url": next_url, "private": private, "allow_execute_sql": await self.ds.permission_allowed( - request.actor, "execute-sql", database, default=True + request.actor, "execute-sql", database_name, default=True ), }, extra_template, ( - f"table-{to_css_class(database)}-{to_css_class(table)}.html", + f"table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", "table.html", ), ) @@ -766,8 +772,8 @@ async def _sql_params_pks(db, table, pk_values): async def display_columns_and_rows( datasette, - database, - table, + database_name, + table_name, description, rows, link_column=False, @@ -776,11 +782,13 @@ async def display_columns_and_rows( ): """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() - db = datasette.databases[database] - table_metadata = datasette.table_metadata(database, table) + db = datasette.databases[database_name] + table_metadata = datasette.table_metadata(database_name, table_name) column_descriptions = table_metadata.get("columns") or {} - column_details = {col.name: col for col in await db.table_column_details(table)} - pks = await db.primary_keys(table) + column_details = { + col.name: col for col in await db.table_column_details(table_name) + } + pks = await db.primary_keys(table_name) pks_for_display = pks if not pks_for_display: pks_for_display = ["rowid"] @@ -805,7 +813,8 @@ async def display_columns_and_rows( ) column_to_foreign_key_table = { - fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table) + fk["column"]: fk["other_table"] + for fk in await db.foreign_keys_for_table(table_name) } cell_rows = [] @@ -826,7 +835,7 @@ async def display_columns_and_rows( "value": markupsafe.Markup( '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format( base_url=base_url, - table_path=datasette.urls.table(database, table), + table_path=datasette.urls.table(database_name, table_name), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) @@ -847,8 +856,8 @@ async def display_columns_and_rows( for candidate in pm.hook.render_cell( value=value, column=column, - table=table, - database=database, + table=table_name, + database=database_name, datasette=datasette, ): candidate = await await_me_maybe(candidate) @@ -862,8 +871,8 @@ async def display_columns_and_rows( display_value = markupsafe.Markup( '<a class="blob-download" href="{}"{}><Binary: {:,} byte{}></a>'.format( datasette.urls.row_blob( - database, - table, + database_name, + table_name, path_from_row_pks(row, pks, not pks), column, ), @@ -883,7 +892,7 @@ async def display_columns_and_rows( link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE display_value = markupsafe.Markup( link_template.format( - database=database, + database=database_name, base_url=base_url, table=tilde_encode(other_table), link_id=tilde_encode(str(value)), From 942411ef946e9a34a2094944d3423cddad27efd3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 26 Apr 2022 15:48:56 -0700 Subject: [PATCH 0694/1866] Execute some TableView queries in parallel Use ?_noparallel=1 to opt out (undocumented, useful for benchmark comparisons) Refs #1723, #1715 --- datasette/views/table.py | 91 +++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d66adb82..23289b29 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,3 +1,4 @@ +import asyncio import itertools import json @@ -5,6 +6,7 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted +from datasette import tracer from datasette.utils import ( await_me_maybe, CustomRow, @@ -150,6 +152,16 @@ class TableView(DataView): default_labels=False, _next=None, _size=None, + ): + with tracer.trace_child_tasks(): + return await self._data_traced(request, default_labels, _next, _size) + + async def _data_traced( + self, + request, + default_labels=False, + _next=None, + _size=None, ): database_route = tilde_decode(request.url_vars["database"]) table_name = tilde_decode(request.url_vars["table"]) @@ -159,6 +171,20 @@ class TableView(DataView): raise NotFound("Database not found: {}".format(database_route)) database_name = db.name + # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather + async def _gather_parallel(*args): + return await asyncio.gather(*args) + + async def _gather_sequential(*args): + results = [] + for fn in args: + results.append(await fn) + return results + + gather = ( + _gather_sequential if request.args.get("_noparallel") else _gather_parallel + ) + # If this is a canned query, not a table, then dispatch to QueryView instead canned_query = await self.ds.get_canned_query( database_name, table_name, request.actor @@ -174,8 +200,12 @@ class TableView(DataView): write=bool(canned_query.get("write")), ) - is_view = bool(await db.get_view_definition(table_name)) - table_exists = bool(await db.table_exists(table_name)) + is_view, table_exists = map( + bool, + await gather( + db.get_view_definition(table_name), db.table_exists(table_name) + ), + ) # If table or view not found, return 404 if not is_view and not table_exists: @@ -497,33 +527,44 @@ class TableView(DataView): ) ) - if not nofacet: - for facet in facet_instances: - ( + async def execute_facets(): + if not nofacet: + # Run them in parallel + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await gather(*facet_awaitables) + for ( instance_facet_results, instance_facets_timed_out, - ) = await facet.facet_results() - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) - # Calculate suggested facets suggested_facets = [] - if ( - self.ds.setting("suggest_facets") - and self.ds.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - for facet in facet_instances: - suggested_facets.extend(await facet.suggest()) + + async def execute_suggested_facets(): + # Calculate suggested facets + if ( + self.ds.setting("suggest_facets") + and self.ds.setting("allow_facet") + and not _next + and not nofacet + and not nosuggest + ): + # Run them in parallel + facet_suggest_awaitables = [ + facet.suggest() for facet in facet_instances + ] + for suggest_result in await gather(*facet_suggest_awaitables): + suggested_facets.extend(suggest_result) + + await gather(execute_facets(), execute_suggested_facets()) # Figure out columns and rows for the query columns = [r[0] for r in results.description] From 94a3171b01fde5c52697aeeff052e3ad4bab5391 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 28 Apr 2022 13:29:11 -0700 Subject: [PATCH 0695/1866] .plugin_config() can return None --- docs/internals.rst | 4 ++++ docs/writing_plugins.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index aad608dc..18822d47 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -288,6 +288,10 @@ All databases are listed, irrespective of user permissions. This means that the This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. +The return value will be the value from the configuration file - usually a dictionary. + +If the plugin is not configured the return value will be ``None``. + .. _datasette_render_template: await .render_template(template, context=None, request=None) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 89f7f5eb..9aee70f6 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -182,6 +182,8 @@ When you are writing plugins, you can access plugin configuration like this usin This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` in the above example. +If there is no configuration for that plugin, the method will return ``None``. + If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so:: { From 4afc1afc721ac0d14f58b0f8339c1bf431d5313c Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 12:13:11 -0700 Subject: [PATCH 0696/1866] Depend on click-default-group-wheel>=1.2.2 Refs #1733 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f0562fd..fcb43aa1 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( install_requires=[ "asgiref>=3.2.10,<3.6.0", "click>=7.1.1,<8.2.0", - "click-default-group~=1.2.2", + "click-default-group-wheel>=1.2.2", "Jinja2>=2.10.3,<3.1.0", "hupper~=1.9", "httpx>=0.20", From 7e03394734307a5761e4c98d902b6a8cab188562 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 12:20:14 -0700 Subject: [PATCH 0697/1866] Optional uvicorn import for Pyodide, refs #1733 --- datasette/app.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d269372c..a5330458 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -26,7 +26,6 @@ from itsdangerous import URLSafeSerializer from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -import uvicorn from .views.base import DatasetteError, ureg from .views.database import DatabaseDownload, DatabaseView @@ -806,6 +805,15 @@ class Datasette: datasette_version = {"version": __version__} if self.version_note: datasette_version["note"] = self.version_note + + try: + # Optional import to avoid breaking Pyodide + # https://github.com/simonw/datasette/issues/1733#issuecomment-1115268245 + import uvicorn + + uvicorn_version = uvicorn.__version__ + except ImportError: + uvicorn_version = None info = { "python": { "version": ".".join(map(str, sys.version_info[:3])), @@ -813,7 +821,7 @@ class Datasette: }, "datasette": datasette_version, "asgi": "3.0", - "uvicorn": uvicorn.__version__, + "uvicorn": uvicorn_version, "sqlite": { "version": sqlite_version, "fts_versions": fts_versions, From 687907aa2b1bde4de6ae7155b0e2a949ca015ca9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 12:39:06 -0700 Subject: [PATCH 0698/1866] Remove python-baseconv dependency, refs #1733, closes #1734 --- datasette/actor_auth_cookie.py | 2 +- datasette/utils/baseconv.py | 59 ++++++++++++++++++++++++++++++++++ docs/authentication.rst | 4 +-- setup.py | 1 - tests/test_auth.py | 2 +- 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 datasette/utils/baseconv.py diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 15ecd331..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,6 +1,6 @@ from datasette import hookimpl from itsdangerous import BadSignature -import baseconv +from datasette.utils import baseconv import time diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py new file mode 100644 index 00000000..27e4fb00 --- /dev/null +++ b/datasette/utils/baseconv.py @@ -0,0 +1,59 @@ +""" +Convert numbers from base 10 integers to base X strings and back again. + +Sample usage: + +>>> base20 = BaseConverter('0123456789abcdefghij') +>>> base20.from_decimal(1234) +'31e' +>>> base20.to_decimal('31e') +1234 + +Originally shared here: https://www.djangosnippets.org/snippets/1431/ +""" + + +class BaseConverter(object): + decimal_digits = "0123456789" + + def __init__(self, digits): + self.digits = digits + + def from_decimal(self, i): + return self.convert(i, self.decimal_digits, self.digits) + + def to_decimal(self, s): + return int(self.convert(s, self.digits, self.decimal_digits)) + + def convert(number, fromdigits, todigits): + # Based on http://code.activestate.com/recipes/111286/ + if str(number)[0] == "-": + number = str(number)[1:] + neg = 1 + else: + neg = 0 + + # make an integer out of the number + x = 0 + for digit in str(number): + x = x * len(fromdigits) + fromdigits.index(digit) + + # create the result in base 'len(todigits)' + if x == 0: + res = todigits[0] + else: + res = "" + while x > 0: + digit = x % len(todigits) + res = todigits[digit] + res + x = int(x / len(todigits)) + if neg: + res = "-" + res + return res + + convert = staticmethod(convert) + + +bin = BaseConverter("01") +hexconv = BaseConverter("0123456789ABCDEF") +base62 = BaseConverter("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz") diff --git a/docs/authentication.rst b/docs/authentication.rst index 24960733..685dab15 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -401,12 +401,12 @@ Including an expiry time ``ds_actor`` cookies can optionally include a signed expiry timestamp, after which the cookies will no longer be valid. Authentication plugins may chose to use this mechanism to limit the lifetime of the cookie. For example, if a plugin implements single-sign-on against another source it may decide to set short-lived cookies so that if the user is removed from the SSO system their existing Datasette cookies will stop working shortly afterwards. -To include an expiry, add a ``"e"`` key to the cookie value containing a `base62-encoded integer <https://pypi.org/project/python-baseconv/>`__ representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours: +To include an expiry, add a ``"e"`` key to the cookie value containing a base62-encoded integer representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours: .. code-block:: python import time - import baseconv + from datasette.utils import baseconv expires_at = int(time.time()) + (24 * 60 * 60) diff --git a/setup.py b/setup.py index fcb43aa1..ca449f02 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,6 @@ setup( "PyYAML>=5.3,<7.0", "mergedeep>=1.1.1,<1.4.0", "itsdangerous>=1.1,<3.0", - "python-baseconv==1.2.2", ], entry_points=""" [console_scripts] diff --git a/tests/test_auth.py b/tests/test_auth.py index 974f89ea..4ef35a76 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,5 +1,5 @@ from .fixtures import app_client -import baseconv +from datasette.utils import baseconv import pytest import time From a29c1277896b6a7905ef5441c42a37bc15f67599 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 12:44:09 -0700 Subject: [PATCH 0699/1866] Rename to_decimal/from_decimal to decode/encode, refs #1734 --- datasette/utils/baseconv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index 27e4fb00..c4b64908 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -19,10 +19,10 @@ class BaseConverter(object): def __init__(self, digits): self.digits = digits - def from_decimal(self, i): + def encode(self, i): return self.convert(i, self.decimal_digits, self.digits) - def to_decimal(self, s): + def decode(self, s): return int(self.convert(s, self.digits, self.decimal_digits)) def convert(number, fromdigits, todigits): From 3f00a29141bdea5be747f6d1c93871ccdb792167 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 13:15:27 -0700 Subject: [PATCH 0700/1866] Clean up compatibility with Pyodide (#1736) * Optional uvicorn import for Pyodide, refs #1733 * --setting num_sql_threads 0 to disable threading, refs #1735 --- datasette/app.py | 11 ++++++++--- datasette/database.py | 19 +++++++++++++++++++ docs/settings.rst | 2 ++ tests/test_internals_datasette.py | 14 +++++++++++++- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index a5330458..b7b84371 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -288,9 +288,12 @@ class Datasette: self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note - self.executor = futures.ThreadPoolExecutor( - max_workers=self.setting("num_sql_threads") - ) + if self.setting("num_sql_threads") == 0: + self.executor = None + else: + self.executor = futures.ThreadPoolExecutor( + max_workers=self.setting("num_sql_threads") + ) self.max_returned_rows = self.setting("max_returned_rows") self.sql_time_limit_ms = self.setting("sql_time_limit_ms") self.page_size = self.setting("default_page_size") @@ -862,6 +865,8 @@ class Datasette: ] def _threads(self): + if self.setting("num_sql_threads") == 0: + return {"num_threads": 0, "threads": []} threads = list(threading.enumerate()) d = { "num_threads": len(threads), diff --git a/datasette/database.py b/datasette/database.py index ba594a8c..44d32667 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -45,6 +45,9 @@ class Database: self._cached_table_counts = None self._write_thread = None self._write_queue = None + # These are used when in non-threaded mode: + self._read_connection = None + self._write_connection = None if not self.is_mutable and not self.is_memory: p = Path(path) self.hash = inspect_hash(p) @@ -134,6 +137,14 @@ class Database: return results async def execute_write_fn(self, fn, block=True): + if self.ds.executor is None: + # non-threaded mode + if self._write_connection is None: + self._write_connection = self.connect(write=True) + self.ds._prepare_connection(self._write_connection, self.name) + return fn(self._write_connection) + + # threaded mode task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") if self._write_queue is None: self._write_queue = queue.Queue() @@ -177,6 +188,14 @@ class Database: task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): + if self.ds.executor is None: + # non-threaded mode + if self._read_connection is None: + self._read_connection = self.connect() + self.ds._prepare_connection(self._read_connection, self.name) + return fn(self._read_connection) + + # threaded mode def in_thread(): conn = getattr(connections, self.name, None) if not conn: diff --git a/docs/settings.rst b/docs/settings.rst index 60c4b36d..8437fb04 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -107,6 +107,8 @@ Maximum number of threads in the thread pool Datasette uses to execute SQLite qu datasette mydatabase.db --setting num_sql_threads 10 +Setting this to 0 turns off threaded SQL queries entirely - useful for environments that do not support threading such as `Pyodide <https://pyodide.org/>`__. + .. _setting_allow_facet: allow_facet diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index cc200a2d..1dc14cab 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,7 +1,7 @@ """ Tests for the datasette.app.Datasette class """ -from datasette.app import Datasette +from datasette.app import Datasette, Database from itsdangerous import BadSignature from .fixtures import app_client import pytest @@ -63,3 +63,15 @@ async def test_datasette_constructor(): "hash": None, } ] + + +@pytest.mark.asyncio +async def test_num_sql_threads_zero(): + ds = Datasette([], memory=True, settings={"num_sql_threads": 0}) + db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) + await db.execute_write("create table t(id integer primary key)") + await db.execute_write("insert into t (id) values (1)") + response = await ds.client.get("/-/threads.json") + assert response.json() == {"num_threads": 0, "threads": []} + response2 = await ds.client.get("/test_num_sql_threads_zero/t.json?_shape=array") + assert response2.json() == [{"id": 1}] From 943aa2e1f7341cb51e60332cde46bde650c64217 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 14:38:34 -0700 Subject: [PATCH 0701/1866] Release 0.62a0 Refs #1683, #1701, #1712, #1717, #1718, #1733 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 02451a1e..cf18c441 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.61.1" +__version__ = "0.62a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 03cf62b6..74814fcb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_62a0: + +0.62a0 (2022-05-02) +------------------- + +- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details. +- Datasette should now be compatible with Pyodide. (:issue:`1733`) +- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__) +- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) +- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) +- Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`) +- Code examples in the documentation are now all formatted using Black. (:issue:`1718`) +- ``Request.fake()`` method is now documented, see :ref:`internals_request`. + .. _v0_61_1: 0.61.1 (2022-03-23) From 847d6b1aac38c3e776e8c600eed07ba4c9ac9942 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 16:32:24 -0700 Subject: [PATCH 0702/1866] Test wheel against Pyodide, refs #1737, #1733 --- .github/workflows/test-pyodide.yml | 28 ++++++++++++++++++ test-in-pyodide-with-shot-scraper.sh | 43 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 .github/workflows/test-pyodide.yml create mode 100755 test-in-pyodide-with-shot-scraper.sh diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml new file mode 100644 index 00000000..3715d055 --- /dev/null +++ b/.github/workflows/test-pyodide.yml @@ -0,0 +1,28 @@ +name: Test in Pyodide with shot-scraper + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + cache: 'pip' + cache-dependency-path: '**/setup.py' + - name: Cache Playwright browsers + uses: actions/cache@v2 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-browsers + - name: Install Playwright dependencies + run: | + pip install shot-scraper + shot-scraper install + - name: Run test + run: | + ./test-in-pyodide-with-shot-scraper.sh diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh new file mode 100755 index 00000000..0f29c0e0 --- /dev/null +++ b/test-in-pyodide-with-shot-scraper.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Build the wheel +python3 -m build + +# Find name of wheel +wheel=$(basename $(ls dist/*.whl)) +# strip off the dist/ + + +# Create a blank index page +echo ' +<script src="https://cdn.jsdelivr.net/pyodide/v0.20.0/full/pyodide.js"></script> +' > dist/index.html + +# Run a server for that dist/ folder +cd dist +python3 -m http.server 8529 & +cd .. + +shot-scraper javascript http://localhost:8529/ " +async () => { + let pyodide = await loadPyodide(); + await pyodide.loadPackage(['micropip', 'ssl', 'setuptools']); + let output = await pyodide.runPythonAsync(\` + import micropip + await micropip.install('h11==0.12.0') + await micropip.install('http://localhost:8529/$wheel') + import ssl + import setuptools + from datasette.app import Datasette + ds = Datasette(memory=True, settings={'num_sql_threads': 0}) + (await ds.client.get('/_memory.json?sql=select+55+as+itworks&_shape=array')).text + \`); + if (JSON.parse(output)[0].itworks != 55) { + throw 'Got ' + output + ', expected itworks: 55'; + } + return 'Test passed!'; +} +" + +# Shut down the server +pkill -f 'http.server 8529' From c0cbcf2aba0d8393ba464acc515803ebf2eeda12 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 16:36:58 -0700 Subject: [PATCH 0703/1866] Tweaks to test scripts, refs #1737 --- .github/workflows/test-pyodide.yml | 2 +- test-in-pyodide-with-shot-scraper.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 3715d055..beb6a5fb 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -21,7 +21,7 @@ jobs: key: ${{ runner.os }}-browsers - name: Install Playwright dependencies run: | - pip install shot-scraper + pip install shot-scraper build shot-scraper install - name: Run test run: | diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 0f29c0e0..e5df7398 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -1,12 +1,12 @@ #!/bin/bash +set -e +# So the script fails if there are any errors # Build the wheel python3 -m build -# Find name of wheel +# Find name of wheel, strip off the dist/ wheel=$(basename $(ls dist/*.whl)) -# strip off the dist/ - # Create a blank index page echo ' From d60f163528f466b1127b2935c3b6869c34fd6545 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 2 May 2022 16:40:49 -0700 Subject: [PATCH 0704/1866] Run on push and PR, closes #1737 --- .github/workflows/test-pyodide.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index beb6a5fb..1b75aade 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -1,6 +1,8 @@ name: Test in Pyodide with shot-scraper on: + push: + pull_request: workflow_dispatch: jobs: From 280ff372ab30df244f6c54f6f3002da57334b3d7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 3 May 2022 07:59:18 -0700 Subject: [PATCH 0705/1866] ETag support for .db downloads, closes #1739 --- datasette/utils/testing.py | 20 ++++++++++++++++++-- datasette/views/database.py | 7 +++++++ tests/test_html.py | 10 +++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 94750b1f..640c94e6 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -55,10 +55,21 @@ class TestClient: @async_to_sync async def get( - self, path, follow_redirects=False, redirect_count=0, method="GET", cookies=None + self, + path, + follow_redirects=False, + redirect_count=0, + method="GET", + cookies=None, + if_none_match=None, ): return await self._request( - path, follow_redirects, redirect_count, method, cookies + path=path, + follow_redirects=follow_redirects, + redirect_count=redirect_count, + method=method, + cookies=cookies, + if_none_match=if_none_match, ) @async_to_sync @@ -110,6 +121,7 @@ class TestClient: headers=None, post_body=None, content_type=None, + if_none_match=None, ): return await self._request( path, @@ -120,6 +132,7 @@ class TestClient: headers=headers, post_body=post_body, content_type=content_type, + if_none_match=if_none_match, ) async def _request( @@ -132,10 +145,13 @@ class TestClient: headers=None, post_body=None, content_type=None, + if_none_match=None, ): headers = headers or {} if content_type: headers["content-type"] = content_type + if if_none_match: + headers["if-none-match"] = if_none_match httpx_response = await self.ds.client.request( method, path, diff --git a/datasette/views/database.py b/datasette/views/database.py index 9a8aca32..bc08ba05 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -183,6 +183,13 @@ class DatabaseDownload(DataView): headers = {} if self.ds.cors: add_cors_headers(headers) + if db.hash: + etag = '"{}"'.format(db.hash) + headers["Etag"] = etag + # Has user seen this already? + if_none_match = request.headers.get("if-none-match") + if if_none_match and if_none_match == etag: + return Response("", status=304) headers["Transfer-Encoding"] = "chunked" return AsgiFileDownload( filepath, diff --git a/tests/test_html.py b/tests/test_html.py index 42f1a3ee..409fec68 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -401,7 +401,7 @@ def test_database_download_for_immutable(): assert len(soup.findAll("a", {"href": re.compile(r"\.db$")})) # Check we can actually download it download_response = client.get("/fixtures.db") - assert 200 == download_response.status + assert download_response.status == 200 # Check the content-length header exists assert "content-length" in download_response.headers content_length = download_response.headers["content-length"] @@ -413,6 +413,14 @@ def test_database_download_for_immutable(): == 'attachment; filename="fixtures.db"' ) assert download_response.headers["transfer-encoding"] == "chunked" + # ETag header should be present and match db.hash + assert "etag" in download_response.headers + etag = download_response.headers["etag"] + assert etag == '"{}"'.format(client.ds.databases["fixtures"].hash) + # Try a second download with If-None-Match: current-etag + download_response2 = client.get("/fixtures.db", if_none_match=etag) + assert download_response2.body == b"" + assert download_response2.status == 304 def test_database_download_disallowed_for_mutable(app_client): From a5acfff4bd364d30ce8878e19f9839890371ef14 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 16 May 2022 17:06:40 -0700 Subject: [PATCH 0706/1866] Empty Datasette([]) list is no longer required --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 1bbaaac1..41046bfb 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -15,7 +15,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl @pytest.mark.asyncio async def test_plugin_is_installed(): - datasette = Datasette([], memory=True) + datasette = Datasette(memory=True) response = await datasette.client.get("/-/plugins.json") assert response.status_code == 200 installed_plugins = {p["name"] for p in response.json()} From 3508bf7875f8d62b2725222f3b07747974d54b97 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 17 May 2022 12:40:05 -0700 Subject: [PATCH 0707/1866] --nolock mode to ignore locked files, closes #1744 --- datasette/app.py | 2 ++ datasette/cli.py | 7 +++++++ datasette/database.py | 2 ++ docs/cli-reference.rst | 1 + docs/getting_started.rst | 4 +++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index b7b84371..f43700d4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -213,6 +213,7 @@ class Datasette: config_dir=None, pdb=False, crossdb=False, + nolock=False, ): assert config_dir is None or isinstance( config_dir, Path @@ -238,6 +239,7 @@ class Datasette: self.databases = collections.OrderedDict() self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb + self.nolock = nolock if memory or crossdb or not self.files: self.add_database(Database(self, is_memory=True), name="_memory") # memory_name is a random string so that each Datasette instance gets its own diff --git a/datasette/cli.py b/datasette/cli.py index 3c6e1b2c..8781747c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -452,6 +452,11 @@ def uninstall(packages, yes): is_flag=True, help="Enable cross-database joins using the /_memory database", ) +@click.option( + "--nolock", + is_flag=True, + help="Ignore locking, open locked files in read-only mode", +) @click.option( "--ssl-keyfile", help="SSL key file", @@ -486,6 +491,7 @@ def serve( open_browser, create, crossdb, + nolock, ssl_keyfile, ssl_certfile, return_instance=False, @@ -545,6 +551,7 @@ def serve( version_note=version_note, pdb=pdb, crossdb=crossdb, + nolock=nolock, ) # if files is a single directory, use that as config_dir= diff --git a/datasette/database.py b/datasette/database.py index 44d32667..fa558045 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -89,6 +89,8 @@ class Database: # mode=ro or immutable=1? if self.is_mutable: qs = "?mode=ro" + if self.ds.nolock: + qs += "&nolock=1" else: qs = "?immutable=1" assert not (write and not self.is_mutable) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2a6fbfc8..1c1aff15 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -115,6 +115,7 @@ datasette serve --help --create Create database files if they do not exist --crossdb Enable cross-database joins using the /_memory database + --nolock Ignore locking, open locked files in read-only mode --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file --help Show this message and exit. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 3e357afb..502a9e5a 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -56,7 +56,9 @@ like so: :: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock + +The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From 5555bc8aef043f75d2200f66de90c54aeeaa08c3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 17 May 2022 12:43:44 -0700 Subject: [PATCH 0708/1866] How to run cog, closes #1745 --- docs/contributing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c193ba49..bddceafe 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -211,6 +211,17 @@ For added productivity, you can use use `sphinx-autobuild <https://pypi.org/proj Now browse to ``http://localhost:8000/`` to view the documentation. Any edits you make should be instantly reflected in your browser. +.. _contributing_documentation_cog: + +Running Cog +~~~~~~~~~~~ + +Some pages of documentation (in particular the :ref:`cli_reference`) are automatically updated using `Cog <https://github.com/nedbat/cog>`__. + +To update these pages, run the following command:: + + cog -r docs/*.rst + .. _contributing_continuous_deployment: Continuously deployed demo instances From b393e164dc9e962702546d6f1ad9c857b5788dc0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 17 May 2022 12:45:28 -0700 Subject: [PATCH 0709/1866] ReST fix --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 502a9e5a..af3a1385 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -58,7 +58,7 @@ like so: datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock -The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. +The ``--nolock`` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From 7d1e004ff679b3fb4dca36d1d751a1ad16688fe6 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 17 May 2022 12:59:28 -0700 Subject: [PATCH 0710/1866] Fix test I broke in #1744 --- tests/test_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index dca65f26..d0f6e26c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,6 +150,7 @@ def test_metadata_yaml(): help_settings=False, pdb=False, crossdb=False, + nolock=False, open_browser=False, create=False, ssl_keyfile=None, From 0e2f6f1f82f4445a63f1251470a7778a34f5c8b9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 18 May 2022 17:37:46 -0700 Subject: [PATCH 0711/1866] datasette-copyable is an example of register_output_renderer --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 3c9ae2e2..c0d88964 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -557,7 +557,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th "atom_updated", }.issubset(columns) -Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__ +Examples: `datasette-atom <https://datasette.io/plugins/datasette-atom>`_, `datasette-ics <https://datasette.io/plugins/datasette-ics>`_, `datasette-geojson <https://datasette.io/plugins/datasette-geojson>`__, `datasette-copyable <https://datasette.io/plugins/datasette-copyable>`__ .. _plugin_register_routes: From 18a6e05887abf1ac946a6e0d36ce662dfd8aeff1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 20 May 2022 12:05:33 -0700 Subject: [PATCH 0712/1866] Added "follow a tutorial" to getting started docs Closes #1747 --- docs/getting_started.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index af3a1385..00b753a9 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,8 @@ Getting started =============== +.. _getting_started_demo: + Play with a live demo --------------------- @@ -9,6 +11,16 @@ The best way to experience Datasette for the first time is with a demo: * `global-power-plants.datasettes.com <https://global-power-plants.datasettes.com/global-power-plants/global-power-plants>`__ provides a searchable database of power plants around the world, using data from the `World Resources Institude <https://www.wri.org/publication/global-power-plant-database>`__ rendered using the `datasette-cluster-map <https://github.com/simonw/datasette-cluster-map>`__ plugin. * `fivethirtyeight.datasettes.com <https://fivethirtyeight.datasettes.com/fivethirtyeight>`__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository <https://github.com/fivethirtyeight/data>`__. +.. _getting_started_tutorial: + +Follow a tutorial +----------------- + +Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you get started with the tool. Try one of the following: + +- `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database. +- `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data. + .. _getting_started_glitch: Try Datasette without installing anything using Glitch From 1465fea4798599eccfe7e8f012bd8d9adfac3039 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 20 May 2022 12:11:08 -0700 Subject: [PATCH 0713/1866] sphinx-copybutton for docs, closes #1748 --- docs/conf.py | 2 +- setup.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d114bc52..351cb1b1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#"), diff --git a/setup.py b/setup.py index ca449f02..aad05840 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,13 @@ setup( """, setup_requires=["pytest-runner"], extras_require={ - "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell", "blacken-docs"], + "docs": [ + "sphinx_rtd_theme", + "sphinx-autobuild", + "codespell", + "blacken-docs", + "sphinx-copybutton", + ], "test": [ "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", From 1d33fd03b3c211e0f48a8f3bde83880af89e4e69 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 20 May 2022 13:34:51 -0700 Subject: [PATCH 0714/1866] Switch docs theme to Furo, refs #1746 --- docs/_static/css/custom.css | 7 ++-- .../layout.html => _static/js/custom.js} | 34 ------------------- docs/_templates/base.html | 6 ++++ docs/_templates/sidebar/brand.html | 16 +++++++++ docs/_templates/sidebar/navigation.html | 11 ++++++ docs/conf.py | 24 +++---------- docs/installation.rst | 1 + docs/plugin_hooks.rst | 1 + setup.py | 2 +- 9 files changed, 45 insertions(+), 57 deletions(-) rename docs/{_templates/layout.html => _static/js/custom.js} (55%) create mode 100644 docs/_templates/base.html create mode 100644 docs/_templates/sidebar/brand.html create mode 100644 docs/_templates/sidebar/navigation.html diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 4dabb725..0a6f8799 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,7 +1,8 @@ a.external { overflow-wrap: anywhere; } - -div .wy-side-nav-search > div.version { - color: rgba(0,0,0,0.75); +body[data-theme="dark"] .sidebar-logo-container { + background-color: white; + padding: 5px; + opacity: 0.6; } diff --git a/docs/_templates/layout.html b/docs/_static/js/custom.js similarity index 55% rename from docs/_templates/layout.html rename to docs/_static/js/custom.js index 785cdc7c..efca33ed 100644 --- a/docs/_templates/layout.html +++ b/docs/_static/js/custom.js @@ -1,35 +1,3 @@ -{%- extends "!layout.html" %} - -{% block htmltitle %} -{{ super() }} -<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script> -{% endblock %} - -{% block sidebartitle %} - -<a href="https://datasette.io/"> - <img src="{{ pathto('_static/' + logo, 1) }}" class="logo" alt="{{ _('Logo') }}"/> -</a> - -{% if theme_display_version %} - {%- set nav_version = version %} - {% if READTHEDOCS and current_version %} - {%- set nav_version = current_version %} - {% endif %} - {% if nav_version %} - <div class="version"> - {{ nav_version }} - </div> - {% endif %} -{% endif %} - -{% include "searchbox.html" %} - -{% endblock %} - -{% block footer %} -{{ super() }} -<script> jQuery(function ($) { // Show banner linking to /stable/ if this is a /latest/ page if (!/\/latest\//.test(location.pathname)) { @@ -57,5 +25,3 @@ jQuery(function ($) { } }); }); -</script> -{% endblock %} diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 00000000..969de5ab --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,6 @@ +{%- extends "!base.html" %} + +{% block site_meta %} +{{ super() }} +<script defer data-domain="docs.datasette.io" src="https://plausible.io/js/plausible.js"></script> +{% endblock %} diff --git a/docs/_templates/sidebar/brand.html b/docs/_templates/sidebar/brand.html new file mode 100644 index 00000000..8be9e8ee --- /dev/null +++ b/docs/_templates/sidebar/brand.html @@ -0,0 +1,16 @@ +<div class="sidebar-brand centered"> + {% block brand_content %} + <div class="sidebar-logo-container"> + <a href="https://datasette.io/"><img class="sidebar-logo" src="{{ logo_url }}" alt="Datasette"></a> + </div> + {%- set nav_version = version %} + {% if READTHEDOCS and current_version %} + {%- set nav_version = current_version %} + {% endif %} + {% if nav_version %} + <div class="version"> + {{ nav_version }} + </div> + {% endif %} + {% endblock brand_content %} +</div> diff --git a/docs/_templates/sidebar/navigation.html b/docs/_templates/sidebar/navigation.html new file mode 100644 index 00000000..c460a17e --- /dev/null +++ b/docs/_templates/sidebar/navigation.html @@ -0,0 +1,11 @@ +<div class="sidebar-tree"> + <ul> + <li class="toctree-l1"><a class="reference internal" href="{{ pathto(master_doc) }}">Contents</a></li> + </ul> + {{ toctree( + collapse=True, + titles_only=False, + maxdepth=3, + includehidden=True, +) }} +</div> \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 351cb1b1..25d2acfe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,18 +90,15 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "logo_only": True, - "style_nav_header_background": "white", - "prev_next_buttons_location": "both", + "sidebar_hide_name": True, } - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". @@ -112,20 +109,9 @@ html_logo = "datasette-logo.svg" html_css_files = [ "css/custom.css", ] - - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - "**": [ - "relations.html", # needs 'show_related': True theme option to display - "searchbox.html", - ] -} - +html_js_files = [ + "js/custom.js" +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/docs/installation.rst b/docs/installation.rst index e8bef9cd..a4757736 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,6 +13,7 @@ If you want to start making contributions to the Datasette project by installing .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _installation_basic: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c0d88964..7d10fe37 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -20,6 +20,7 @@ For example, you can implement the ``render_cell`` plugin hook like this even th .. contents:: List of plugin hooks :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _plugin_hook_prepare_connection: diff --git a/setup.py b/setup.py index aad05840..d3fcdbd1 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "sphinx_rtd_theme", + "furo==2022.4.7", "sphinx-autobuild", "codespell", "blacken-docs", From 4446075334ea7231beb56b630bc7ec363afc2d08 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 20 May 2022 13:44:23 -0700 Subject: [PATCH 0715/1866] Append warning to the write element, refs #1746 --- docs/_static/js/custom.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js index efca33ed..91c3e306 100644 --- a/docs/_static/js/custom.js +++ b/docs/_static/js/custom.js @@ -17,11 +17,7 @@ jQuery(function ($) { </div>` ); warning.find("a").attr("href", stableUrl); - var body = $("div.body"); - if (!body.length) { - body = $("div.document"); - } - body.prepend(warning); + $("article[role=main]").prepend(warning); } }); }); From b010af7bb85856aeb44f69e7e980f617c1fc0db1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 20 May 2022 15:23:09 -0700 Subject: [PATCH 0716/1866] Updated copyright years in documentation footer --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 25d2acfe..7ffeedd0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ master_doc = "index" # General information about the project. project = "Datasette" -copyright = "2017-2021, Simon Willison" +copyright = "2017-2022, Simon Willison" author = "Simon Willison" # Disable -- turning into – From adedd85b68ec66e03b97fb62ff4da8987734436e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 28 May 2022 18:42:31 -0700 Subject: [PATCH 0717/1866] Clarify that request.headers names are converted to lowercase --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 18822d47..da135282 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming The request scheme - usually ``https`` or ``http``. ``.headers`` - dictionary (str -> str) - A dictionary of incoming HTTP request headers. + A dictionary of incoming HTTP request headers. Header names have been converted to lowercase. ``.cookies`` - dictionary (str -> str) A dictionary of incoming cookies From 8dd816bc76937f1e37f86acce10dc2cb4fa31e52 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 30 May 2022 15:42:38 -0700 Subject: [PATCH 0718/1866] Applied Black --- docs/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7ffeedd0..4ef6b768 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -109,9 +109,7 @@ html_logo = "datasette-logo.svg" html_css_files = [ "css/custom.css", ] -html_js_files = [ - "js/custom.js" -] +html_js_files = ["js/custom.js"] # -- Options for HTMLHelp output ------------------------------------------ From 2e9751672d4fe329b3c359d5b7b1992283185820 Mon Sep 17 00:00:00 2001 From: Naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Tue, 31 May 2022 14:28:40 -0500 Subject: [PATCH 0719/1866] chore: Set permissions for GitHub actions (#1740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the GitHub token permissions only to the required ones; this way, even if the attackers will succeed in compromising your workflow, they won’t be able to do much. - Included permissions for the action. https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/deploy-latest.yml | 3 +++ .github/workflows/prettier.yml | 3 +++ .github/workflows/publish.yml | 3 +++ .github/workflows/push_docker_tag.yml | 3 +++ .github/workflows/spellcheck.yml | 3 +++ .github/workflows/test-coverage.yml | 3 +++ .github/workflows/test-pyodide.yml | 3 +++ .github/workflows/test.yml | 3 +++ .github/workflows/tmate-mac.yml | 3 +++ .github/workflows/tmate.yml | 3 +++ 10 files changed, 30 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index a61f6629..2b94a7f1 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -5,6 +5,9 @@ on: branches: - main +permissions: + contents: read + jobs: deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 9dfe7ee0..ded41040 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -2,6 +2,9 @@ name: Check JavaScript for conformance with Prettier on: [push] +permissions: + contents: read + jobs: prettier: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3e4f8146..9ef09d2e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,9 @@ on: release: types: [created] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index 9a3969f0..afe8d6b2 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -6,6 +6,9 @@ on: version_tag: description: Tag to build and push +permissions: + contents: read + jobs: deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 2e24d3eb..a2621ecc 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -2,6 +2,9 @@ name: Check spelling in documentation on: [push, pull_request] +permissions: + contents: read + jobs: spellcheck: runs-on: ubuntu-latest diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1d1cf332..bd720664 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -7,6 +7,9 @@ on: pull_request: branches: - main +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 1b75aade..bc9593a8 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -5,6 +5,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d916e49..90b6555e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: [push, pull_request] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index 46be117e..fcee0f21 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -3,6 +3,9 @@ name: tmate session mac on: workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: macos-latest diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 02e7bd33..9792245d 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -3,6 +3,9 @@ name: tmate session on: workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest From e780b2f5d662ef3579d801d33567440055d4e84d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 20 Jun 2022 10:54:23 -0700 Subject: [PATCH 0720/1866] Trying out one-sentence-per-line As suggested here: https://sive.rs/1s Markdown and reStructuredText will display this as if it is a single paragraph, even though the sentences themselves are separated by newlines. This could result in more useful diffs. Trying it out on this page first. --- docs/facets.rst | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/facets.rst b/docs/facets.rst index 0228aa84..2a2eb039 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -3,7 +3,9 @@ Facets ====== -Datasette facets can be used to add a faceted browse interface to any database table. With facets, tables are displayed along with a summary showing the most common values in specified columns. These values can be selected to further filter the table. +Datasette facets can be used to add a faceted browse interface to any database table. +With facets, tables are displayed along with a summary showing the most common values in specified columns. +These values can be selected to further filter the table. .. image:: facets.png @@ -12,11 +14,13 @@ Facets can be specified in two ways: using query string parameters, or in ``meta Facets in query strings ----------------------- -To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL. For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this:: +To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL. +For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this:: /dbname/tablename?_facet=state&_facet=city_id -This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this: +This works for both the HTML interface and the ``.json`` view. +When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this: .. code-block:: json @@ -86,7 +90,8 @@ This works for both the HTML interface and the ``.json`` view. When enabled, fac If Datasette detects that a column is a foreign key, the ``"label"`` property will be automatically derived from the detected label column on the referenced table. -The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). +The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. +You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). .. _facets_metadata: @@ -137,12 +142,14 @@ For the currently filtered data are there any columns which, if applied as a fac * Will return less unique options than the total number of filtered rows * And the query used to evaluate this criteria can be completed in under 50ms -That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries. This means suggested facets are unlikely to appear for tables with millions of records in them. +That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries. +This means suggested facets are unlikely to appear for tables with millions of records in them. Speeding up facets with indexes ------------------------------- -The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: +The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. +Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: $ sqlite3 mydatabase.db SQLite version 3.19.3 2017-06-27 16:48:08 @@ -169,6 +176,7 @@ Example here: `latest.datasette.io/fixtures/facetable?_facet_array=tags <https:/ Facet by date ------------- -If Datasette finds any columns that contain dates in the first 100 values, it will offer a faceting interface against the dates of those values. This works especially well against timestamp values such as ``2019-03-01 12:44:00``. +If Datasette finds any columns that contain dates in the first 100 values, it will offer a faceting interface against the dates of those values. +This works especially well against timestamp values such as ``2019-03-01 12:44:00``. Example here: `latest.datasette.io/fixtures/facetable?_facet_date=created <https://latest.datasette.io/fixtures/facetable?_facet_date=created>`__ From 00e59ec461dc0150772b999c7cc15fcb9b507d58 Mon Sep 17 00:00:00 2001 From: "M. Nasimul Haque" <nasim.haque@gmail.com> Date: Mon, 20 Jun 2022 19:05:44 +0100 Subject: [PATCH 0721/1866] Extract facet pieces of table.html into included templates Thanks, @nsmgr8 --- datasette/templates/_facet_results.html | 28 ++++++++++++++++++ datasette/templates/_suggested_facets.html | 3 ++ datasette/templates/table.html | 33 ++-------------------- 3 files changed, 33 insertions(+), 31 deletions(-) create mode 100644 datasette/templates/_facet_results.html create mode 100644 datasette/templates/_suggested_facets.html diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html new file mode 100644 index 00000000..d0cbcf77 --- /dev/null +++ b/datasette/templates/_facet_results.html @@ -0,0 +1,28 @@ +<div class="facet-results"> + {% for facet_info in sorted_facet_results %} + <div class="facet-info facet-{{ database|to_css_class }}-{{ table|to_css_class }}-{{ facet_info.name|to_css_class }}" id="facet-{{ facet_info.name|to_css_class }}" data-column="{{ facet_info.name }}"> + <p class="facet-info-name"> + <strong>{{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} + <span class="facet-info-total">{% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}</span> + </strong> + {% if facet_info.hideable %} + <a href="{{ facet_info.toggle_url }}" class="cross">✖</a> + {% endif %} + </p> + <ul class="tight-bullets"> + {% for facet_value in facet_info.results %} + {% if not facet_value.selected %} + <li><a href="{{ facet_value.toggle_url }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li> + {% else %} + <li>{{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">✖</a></li> + {% endif %} + {% endfor %} + {% if facet_info.truncated %} + <li class="facet-truncated">{% if request.args._facet_size != "max" -%} + <a href="{{ path_with_replaced_args(request, {"_facet_size": "max"}) }}">…</a>{% else -%}…{% endif %} + </li> + {% endif %} + </ul> + </div> + {% endfor %} +</div> diff --git a/datasette/templates/_suggested_facets.html b/datasette/templates/_suggested_facets.html new file mode 100644 index 00000000..ec98fb36 --- /dev/null +++ b/datasette/templates/_suggested_facets.html @@ -0,0 +1,3 @@ +<p class="suggested-facets"> + Suggested facets: {% for facet in suggested_facets %}<a href="{{ facet.toggle_url }}#facet-{{ facet.name|to_css_class }}">{{ facet.name }}</a>{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} +</p> diff --git a/datasette/templates/table.html b/datasette/templates/table.html index a9e88330..a86398ea 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -142,9 +142,7 @@ <p class="export-links">This data as {% for name, url in renderers.items() %}<a href="{{ url }}">{{ name }}</a>{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, <a href="{{ url_csv }}">CSV</a> (<a href="#export">advanced</a>){% endif %}</p> {% if suggested_facets %} - <p class="suggested-facets"> - Suggested facets: {% for facet in suggested_facets %}<a href="{{ facet.toggle_url }}#facet-{{ facet.name|to_css_class }}">{{ facet.name }}</a>{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} - </p> + {% include "_suggested_facets.html" %} {% endif %} {% if facets_timed_out %} @@ -152,34 +150,7 @@ {% endif %} {% if facet_results %} - <div class="facet-results"> - {% for facet_info in sorted_facet_results %} - <div class="facet-info facet-{{ database|to_css_class }}-{{ table|to_css_class }}-{{ facet_info.name|to_css_class }}" id="facet-{{ facet_info.name|to_css_class }}" data-column="{{ facet_info.name }}"> - <p class="facet-info-name"> - <strong>{{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} - <span class="facet-info-total">{% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}</span> - </strong> - {% if facet_info.hideable %} - <a href="{{ facet_info.toggle_url }}" class="cross">✖</a> - {% endif %} - </p> - <ul class="tight-bullets"> - {% for facet_value in facet_info.results %} - {% if not facet_value.selected %} - <li><a href="{{ facet_value.toggle_url }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li> - {% else %} - <li>{{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">✖</a></li> - {% endif %} - {% endfor %} - {% if facet_info.truncated %} - <li class="facet-truncated">{% if request.args._facet_size != "max" -%} - <a href="{{ path_with_replaced_args(request, {"_facet_size": "max"}) }}">…</a>{% else -%}…{% endif %} - </li> - {% endif %} - </ul> - </div> - {% endfor %} - </div> + {% include "_facet_results.html" %} {% endif %} {% include custom_table_templates %} From 9f1eb0d4eac483b953392157bd9fd6cc4df37de7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jun 2022 10:40:24 -0700 Subject: [PATCH 0722/1866] Bump black from 22.1.0 to 22.6.0 (#1763) Bumps [black](https://github.com/psf/black) from 22.1.0 to 22.6.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.1.0...22.6.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3fcdbd1..29cb77bf 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.17,<0.19", "beautifulsoup4>=4.8.1,<4.12.0", - "black==22.1.0", + "black==22.6.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2,<2.2", "trustme>=0.7,<0.10", From 6373bb341457e5becfd5b67792ac2c8b9ed7c384 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 7 Jul 2022 09:30:49 -0700 Subject: [PATCH 0723/1866] Expose current SQLite row to render_cell hook, closes #1300 --- datasette/hookspecs.py | 2 +- datasette/views/database.py | 1 + datasette/views/table.py | 1 + docs/plugin_hooks.rst | 9 ++++++--- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 5 +++-- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 8f4fecab..c84db0a3 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -60,7 +60,7 @@ def publish_subcommand(publish): @hookspec -def render_cell(value, column, table, database, datasette): +def render_cell(row, value, column, table, database, datasette): """Customize rendering of HTML table cell values""" diff --git a/datasette/views/database.py b/datasette/views/database.py index bc08ba05..42058752 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -375,6 +375,7 @@ class QueryView(DataView): # pylint: disable=no-member plugin_display_value = None for candidate in pm.hook.render_cell( + row=row, value=value, column=column, table=None, diff --git a/datasette/views/table.py b/datasette/views/table.py index 23289b29..cd4be823 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -895,6 +895,7 @@ async def display_columns_and_rows( # pylint: disable=no-member plugin_display_value = None for candidate in pm.hook.render_cell( + row=row, value=value, column=column, table=table_name, diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 7d10fe37..f5c3ee83 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -373,12 +373,15 @@ Examples: `datasette-publish-fly <https://datasette.io/plugins/datasette-publish .. _plugin_hook_render_cell: -render_cell(value, column, table, database, datasette) ------------------------------------------------------- +render_cell(row, value, column, table, database, datasette) +----------------------------------------------------------- Lets you customize the display of values within table cells in the HTML table view. -``value`` - string, integer or None +``row`` - ``sqlite.Row`` + The SQLite row object that the value being rendered is part of + +``value`` - string, integer, float, bytes or None The value that was loaded from the database ``column`` - string diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 1c9b0575..53613b7d 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -98,12 +98,13 @@ def extra_body_script( @hookimpl -def render_cell(value, column, table, database, datasette): +def render_cell(row, value, column, table, database, datasette): async def inner(): # Render some debug output in cell with value RENDER_CELL_DEMO if value == "RENDER_CELL_DEMO": return json.dumps( { + "row": dict(row), "column": column, "table": table, "database": database, diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 15bde962..4a7ad7c6 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -181,12 +181,13 @@ def test_hook_render_cell_demo(app_client): response = app_client.get("/fixtures/simple_primary_key?id=4") soup = Soup(response.body, "html.parser") td = soup.find("td", {"class": "col-content"}) - assert { + assert json.loads(td.string) == { + "row": {"id": "4", "content": "RENDER_CELL_DEMO"}, "column": "content", "table": "simple_primary_key", "database": "fixtures", "config": {"depth": "table", "special": "this-is-simple_primary_key"}, - } == json.loads(td.string) + } @pytest.mark.parametrize( From 035dc5e7b95142d4a700819a8cc4ff64aefe4efe Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 9 Jul 2022 10:25:37 -0700 Subject: [PATCH 0724/1866] More than 90 plugins now --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 9aee70f6..01ee8c90 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -5,7 +5,7 @@ Writing plugins You can write one-off plugins that apply to just one Datasette instance, or you can write plugins which can be installed using ``pip`` and can be shipped to the Python Package Index (`PyPI <https://pypi.org/>`__) for other people to install. -Want to start by looking at an example? The `Datasette plugins directory <https://datasette.io/plugins>`__ lists more than 50 open source plugins with code you can explore. The :ref:`plugin hooks <plugin_hooks>` page includes links to example plugins for each of the documented hooks. +Want to start by looking at an example? The `Datasette plugins directory <https://datasette.io/plugins>`__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks <plugin_hooks>` page includes links to example plugins for each of the documented hooks. .. _writing_plugins_one_off: From 5d76c1f81b2d978f48b85c70d041a2142cf8ee26 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 14 Jul 2022 15:03:33 -0700 Subject: [PATCH 0725/1866] Discord badge Refs https://github.com/simonw/datasette.io/issues/112 --- README.md | 1 + docs/index.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 557d9290..c57ee604 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) +[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw) *An open source multi-tool for exploring and publishing data* diff --git a/docs/index.rst b/docs/index.rst index a2888822..62ed70f8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,8 @@ datasette| :target: https://github.com/simonw/datasette/blob/main/LICENSE .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue :target: https://hub.docker.com/r/datasetteproject/datasette +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord + :target: https://discord.gg/ktd74dm5mw *An open source multi-tool for exploring and publishing data* From c133545fe9c7ac2d509e55bf4bf6164bfbe892ad Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 14 Jul 2022 15:04:38 -0700 Subject: [PATCH 0726/1866] Make discord badge lowercase Refs https://github.com/simonw/datasette.io/issues/112 --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c57ee604..032180aa 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) -[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/ktd74dm5mw) *An open source multi-tool for exploring and publishing data* diff --git a/docs/index.rst b/docs/index.rst index 62ed70f8..051898b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ datasette| :target: https://github.com/simonw/datasette/blob/main/LICENSE .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue :target: https://hub.docker.com/r/datasetteproject/datasette -.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord :target: https://discord.gg/ktd74dm5mw *An open source multi-tool for exploring and publishing data* From 950cc7677f65aa2543067b3bbfc2b6acb98b62c8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 14 Jul 2022 15:18:28 -0700 Subject: [PATCH 0727/1866] Fix missing Discord image Refs https://github.com/simonw/datasette.io/issues/112 --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 051898b1..efe196b3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ Datasette ========= |PyPI| |Changelog| |Python 3.x| |Tests| |License| |docker: -datasette| +datasette| |discord| .. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg :target: https://pypi.org/project/datasette/ From 8188f55efc0fcca1be692b0d0c875f2d1ee99f17 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 15:24:16 -0700 Subject: [PATCH 0728/1866] Rename handle_500 to handle_exception, refs #1770 --- datasette/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f43700d4..43e60dbc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1275,7 +1275,7 @@ class DatasetteRouter: except NotFound as exception: return await self.handle_404(request, send, exception) except Exception as exception: - return await self.handle_500(request, send, exception) + return await self.handle_exception(request, send, exception) async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding @@ -1354,7 +1354,7 @@ class DatasetteRouter: view_name="page", ) except NotFoundExplicit as e: - await self.handle_500(request, send, e) + await self.handle_exception(request, send, e) return # Pull content-type out into separate parameter content_type = "text/html; charset=utf-8" @@ -1369,9 +1369,9 @@ class DatasetteRouter: content_type=content_type, ) else: - await self.handle_500(request, send, exception or NotFound("404")) + await self.handle_exception(request, send, exception or NotFound("404")) - async def handle_500(self, request, send, exception): + async def handle_exception(self, request, send, exception): if self.ds.pdb: import pdb From c09c53f3455a7b9574cf7695478f2b87d20897db Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 16:24:39 -0700 Subject: [PATCH 0729/1866] New handle_exception plugin hook, refs #1770 Also refs: - https://github.com/simonw/datasette-sentry/issues/1 - https://github.com/simonw/datasette-show-errors/issues/2 --- datasette/app.py | 97 +++++++++-------------------------- datasette/forbidden.py | 20 ++++++++ datasette/handle_exception.py | 74 ++++++++++++++++++++++++++ datasette/hookspecs.py | 5 ++ datasette/plugins.py | 2 + docs/plugin_hooks.rst | 78 ++++++++++++++++++++-------- tests/fixtures.py | 1 + tests/plugins/my_plugin_2.py | 18 +++++++ tests/test_permissions.py | 1 + tests/test_plugins.py | 14 +++++ 10 files changed, 215 insertions(+), 95 deletions(-) create mode 100644 datasette/forbidden.py create mode 100644 datasette/handle_exception.py diff --git a/datasette/app.py b/datasette/app.py index 43e60dbc..edd05bb3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -16,7 +16,6 @@ import re import secrets import sys import threading -import traceback import urllib.parse from concurrent import futures from pathlib import Path @@ -27,7 +26,7 @@ from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from .views.base import DatasetteError, ureg +from .views.base import ureg from .views.database import DatabaseDownload, DatabaseView from .views.index import IndexView from .views.special import ( @@ -49,7 +48,6 @@ from .utils import ( PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, call_with_supported_arguments, @@ -87,11 +85,6 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -try: - import rich -except ImportError: - rich = None - app_root = Path(__file__).parent.parent # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 @@ -1274,6 +1267,16 @@ class DatasetteRouter: return except NotFound as exception: return await self.handle_404(request, send, exception) + except Forbidden as exception: + # Try the forbidden() plugin hook + for custom_response in pm.hook.forbidden( + datasette=self.ds, request=request, message=exception.args[0] + ): + custom_response = await await_me_maybe(custom_response) + assert ( + custom_response + ), "Default forbidden() hook should have been called" + return await custom_response.asgi_send(send) except Exception as exception: return await self.handle_exception(request, send, exception) @@ -1372,72 +1375,20 @@ class DatasetteRouter: await self.handle_exception(request, send, exception or NotFound("404")) async def handle_exception(self, request, send, exception): - if self.ds.pdb: - import pdb + responses = [] + for hook in pm.hook.handle_exception( + datasette=self.ds, + request=request, + exception=exception, + ): + response = await await_me_maybe(hook) + if response is not None: + responses.append(response) - pdb.post_mortem(exception.__traceback__) - - if rich is not None: - rich.get_console().print_exception(show_locals=True) - - title = None - if isinstance(exception, Forbidden): - status = 403 - info = {} - message = exception.args[0] - # Try the forbidden() plugin hook - for custom_response in pm.hook.forbidden( - datasette=self.ds, request=request, message=message - ): - custom_response = await await_me_maybe(custom_response) - if custom_response is not None: - await custom_response.asgi_send(send) - return - elif isinstance(exception, Base400): - status = exception.status - info = {} - message = exception.args[0] - elif isinstance(exception, DatasetteError): - status = exception.status - info = exception.error_dict - message = exception.message - if exception.message_is_html: - message = Markup(message) - title = exception.title - else: - status = 500 - info = {} - message = str(exception) - traceback.print_exc() - templates = [f"{status}.html", "error.html"] - info.update( - { - "ok": False, - "error": message, - "status": status, - "title": title, - } - ) - headers = {} - if self.ds.cors: - add_cors_headers(headers) - if request.path.split("?")[0].endswith(".json"): - await asgi_send_json(send, info, status=status, headers=headers) - else: - template = self.ds.jinja_env.select_template(templates) - await asgi_send_html( - send, - await template.render_async( - dict( - info, - urls=self.ds.urls, - app_css_hash=self.ds.app_css_hash(), - menu_links=lambda: [], - ) - ), - status=status, - headers=headers, - ) + assert responses, "Default exception handler should have returned something" + # Even if there are multiple responses use just the first one + response = responses[0] + await response.asgi_send(send) _cleaner_task_str_re = re.compile(r"\S*site-packages/") diff --git a/datasette/forbidden.py b/datasette/forbidden.py new file mode 100644 index 00000000..156a44d4 --- /dev/null +++ b/datasette/forbidden.py @@ -0,0 +1,20 @@ +from os import stat +from datasette import hookimpl, Response + + +@hookimpl(trylast=True) +def forbidden(datasette, request, message): + async def inner(): + return Response.html( + await datasette.render_template( + "error.html", + { + "title": "Forbidden", + "error": message, + }, + request=request, + ), + status=403, + ) + + return inner diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py new file mode 100644 index 00000000..8b7e83e3 --- /dev/null +++ b/datasette/handle_exception.py @@ -0,0 +1,74 @@ +from datasette import hookimpl, Response +from .utils import await_me_maybe, add_cors_headers +from .utils.asgi import ( + Base400, + Forbidden, +) +from .views.base import DatasetteError +from markupsafe import Markup +import pdb +import traceback +from .plugins import pm + +try: + import rich +except ImportError: + rich = None + + +@hookimpl(trylast=True) +def handle_exception(datasette, request, exception): + async def inner(): + if datasette.pdb: + pdb.post_mortem(exception.__traceback__) + + if rich is not None: + rich.get_console().print_exception(show_locals=True) + + title = None + if isinstance(exception, Base400): + status = exception.status + info = {} + message = exception.args[0] + elif isinstance(exception, DatasetteError): + status = exception.status + info = exception.error_dict + message = exception.message + if exception.message_is_html: + message = Markup(message) + title = exception.title + else: + status = 500 + info = {} + message = str(exception) + traceback.print_exc() + templates = [f"{status}.html", "error.html"] + info.update( + { + "ok": False, + "error": message, + "status": status, + "title": title, + } + ) + headers = {} + if datasette.cors: + add_cors_headers(headers) + if request.path.split("?")[0].endswith(".json"): + return Response.json(info, status=status, headers=headers) + else: + template = datasette.jinja_env.select_template(templates) + return Response.html( + await template.render_async( + dict( + info, + urls=datasette.urls, + app_css_hash=datasette.app_css_hash(), + menu_links=lambda: [], + ) + ), + status=status, + headers=headers, + ) + + return inner diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index c84db0a3..a5fb536f 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -138,3 +138,8 @@ def database_actions(datasette, actor, database, request): @hookspec def skip_csrf(datasette, scope): """Mechanism for skipping CSRF checks for certain requests""" + + +@hookspec +def handle_exception(datasette, request, exception): + """Handle an uncaught exception. Can return a Response or None.""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 76b46a47..fef0c8e9 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -15,6 +15,8 @@ DEFAULT_PLUGINS = ( "datasette.default_magic_parameters", "datasette.blob_renderer", "datasette.default_menu_links", + "datasette.handle_exception", + "datasette.forbidden", ) pm = pluggy.PluginManager("datasette") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f5c3ee83..6020a941 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -107,8 +107,8 @@ Extra template variables that should be made available in the rendered template ``view_name`` - string The name of the view being displayed. (``index``, ``database``, ``table``, and ``row`` are the most important ones.) -``request`` - object or None - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` @@ -504,7 +504,7 @@ When a request is received, the ``"render"`` callback function is called with ze The table or view, if one is being rendered. ``request`` - :ref:`internals_request` - The incoming HTTP request. + The current HTTP request. ``view_name`` - string The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones. @@ -599,8 +599,8 @@ The optional view function arguments are as follows: ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - Request object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. ``scope`` - dictionary The incoming ASGI scope dictionary. @@ -947,8 +947,8 @@ actor_from_request(datasette, request) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. This is part of Datasette's :ref:`authentication and permissions system <authentication>`. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request. @@ -1010,8 +1010,8 @@ Example: `datasette-auth-tokens <https://datasette.io/plugins/datasette-auth-tok filters_from_request(request, database, table, datasette) --------------------------------------------------------- -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. ``database`` - string The name of the database. @@ -1178,8 +1178,8 @@ forbidden(datasette, request, message) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. ``message`` - string A message hinting at why the request was forbidden. @@ -1206,21 +1206,55 @@ The function can alternatively return an awaitable function if it needs to make .. code-block:: python - from datasette import hookimpl - from datasette.utils.asgi import Response + from datasette import hookimpl, Response @hookimpl def forbidden(datasette): async def inner(): return Response.html( - await datasette.render_template( - "forbidden.html" - ) + await datasette.render_template("render_message.html", request=request) ) return inner +.. _plugin_hook_handle_exception: + +handle_exception(datasette, request, exception) +----------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``exception`` - ``Exception`` + The exception that was raised. + +This hook is called any time an unexpected exception is raised. You can use it to record the exception. + +If your handler returns a ``Response`` object it will be returned to the client in place of the default Datasette error page. + +The handler can return a response directly, or it can return return an awaitable function that returns a response. + +This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a custom error page: + +.. code-block:: python + + from datasette import hookimpl, Response + import sentry_sdk + + + @hookimpl + def handle_exception(datasette, exception): + sentry_sdk.capture_exception(exception) + async def inner(): + return Response.html( + await datasette.render_template("custom_error.html", request=request) + ) + return inner + .. _plugin_hook_menu_links: menu_links(datasette, actor, request) @@ -1232,8 +1266,8 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor <authentication_actor>`. -``request`` - object or None - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` + The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1281,8 +1315,8 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - object - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` + The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. @@ -1325,8 +1359,8 @@ database_actions(datasette, actor, database, request) ``database`` - string The name of the database. -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. diff --git a/tests/fixtures.py b/tests/fixtures.py index e0e4ec7b..c145ac78 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -68,6 +68,7 @@ EXPECTED_PLUGINS = [ "canned_queries", "extra_js_urls", "extra_template_vars", + "handle_exception", "menu_links", "permission_allowed", "register_routes", diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index f5ce36b3..4df02343 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -185,3 +185,21 @@ def register_routes(datasette): # Also serves to demonstrate over-ride of default paths: (r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", new_table), ] + + +@hookimpl +def handle_exception(datasette, request, exception): + datasette._exception_hook_fired = (request, exception) + if request.args.get("_custom_error"): + return Response.text("_custom_error") + elif request.args.get("_custom_error_async"): + + async def inner(): + return Response.text("_custom_error_async") + + return inner + + +@hookimpl(specname="register_routes") +def register_triger_error(): + return ((r"/trigger-error", lambda: 1 / 0),) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f4169dbe..2a519e76 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -332,6 +332,7 @@ def test_permissions_debug(app_client): assert checks == [ {"action": "permissions-debug", "result": True, "used_default": False}, {"action": "view-instance", "result": None, "used_default": True}, + {"action": "debug-menu", "result": False, "used_default": True}, {"action": "permissions-debug", "result": False, "used_default": True}, {"action": "view-instance", "result": None, "used_default": True}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4a7ad7c6..948a40b8 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -824,6 +824,20 @@ def test_hook_forbidden(restore_working_directory): assert "view-database" == client.ds._last_forbidden_message +def test_hook_handle_exception(app_client): + app_client.get("/trigger-error?x=123") + assert hasattr(app_client.ds, "_exception_hook_fired") + request, exception = app_client.ds._exception_hook_fired + assert request.url == "http://localhost/trigger-error?x=123" + assert isinstance(exception, ZeroDivisionError) + + +@pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) +def test_hook_handle_exception_custom_response(app_client, param): + response = app_client.get("/trigger-error?{}=1".format(param)) + assert response.text == param + + def test_hook_menu_links(app_client): def get_menu_links(html): soup = Soup(html, "html.parser") From 58fd1e33ec7ac5ed85431d5c86d60600cd5280fb Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 16:30:58 -0700 Subject: [PATCH 0730/1866] Hint that you can render templates for these hooks, refs #1770 --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 6020a941..b4869606 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1176,7 +1176,7 @@ forbidden(datasette, request, message) -------------------------------------- ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries. ``request`` - :ref:`internals_request` The current HTTP request. @@ -1224,7 +1224,7 @@ handle_exception(datasette, request, exception) ----------------------------------------------- ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries. ``request`` - :ref:`internals_request` The current HTTP request. From e543a095cc4c1ca895b082cfd1263ca25203a7c0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 17:57:41 -0700 Subject: [PATCH 0731/1866] Updated default plugins in docs, refs #1770 --- docs/plugins.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index f2ed02f7..29078054 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -172,6 +172,24 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "filters_from_request" ] }, + { + "name": "datasette.forbidden", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "forbidden" + ] + }, + { + "name": "datasette.handle_exception", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "handle_exception" + ] + }, { "name": "datasette.publish.cloudrun", "static": false, From 6d5e1955470424cf4faf5d35788d328ebdd6d463 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 17:59:20 -0700 Subject: [PATCH 0732/1866] Release 0.62a1 Refs #1300, #1739, #1744, #1746, #1748, #1759, #1770 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index cf18c441..86f4cf7e 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62a0" +__version__ = "0.62a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 74814fcb..3f105811 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_62a1: + +0.62a1 (2022-07-17) +------------------- + +- New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`) +- The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`) +- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) +- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`) +- Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__. +- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) +- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) +- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__) + .. _v0_62a0: 0.62a0 (2022-05-02) From ed1ebc0f1d4153e3e0934f2af19f82e5fdf137d3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 17 Jul 2022 18:03:33 -0700 Subject: [PATCH 0733/1866] Run blacken-docs, refs #1770 --- docs/plugin_hooks.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b4869606..aec1df56 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1213,7 +1213,9 @@ The function can alternatively return an awaitable function if it needs to make def forbidden(datasette): async def inner(): return Response.html( - await datasette.render_template("render_message.html", request=request) + await datasette.render_template( + "render_message.html", request=request + ) ) return inner @@ -1249,10 +1251,14 @@ This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a @hookimpl def handle_exception(datasette, exception): sentry_sdk.capture_exception(exception) + async def inner(): return Response.html( - await datasette.render_template("custom_error.html", request=request) + await datasette.render_template( + "custom_error.html", request=request + ) ) + return inner .. _plugin_hook_menu_links: From ea6161f8475d9fa41c4879049511c58f692cce04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 18:06:26 -0700 Subject: [PATCH 0734/1866] Bump furo from 2022.4.7 to 2022.6.21 (#1760) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.4.7 to 2022.6.21. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.04.07...2022.06.21) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29cb77bf..558b5c87 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.4.7", + "furo==2022.6.21", "sphinx-autobuild", "codespell", "blacken-docs", From 22354c48ce4d514d7a1b321e5651c7f1340e3f5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 18:06:37 -0700 Subject: [PATCH 0735/1866] Update pytest-asyncio requirement from <0.19,>=0.17 to >=0.17,<0.20 (#1769) Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Changelog](https://github.com/pytest-dev/pytest-asyncio/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.17.0...v0.19.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 558b5c87..a1c51d0b 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ setup( "test": [ "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.17,<0.19", + "pytest-asyncio>=0.17,<0.20", "beautifulsoup4>=4.8.1,<4.12.0", "black==22.6.0", "blacken-docs==1.12.1", From 01369176b0a8943ab45292ffc6f9c929b80a00e8 Mon Sep 17 00:00:00 2001 From: Chris Amico <eyeseast@gmail.com> Date: Sun, 17 Jul 2022 21:12:45 -0400 Subject: [PATCH 0736/1866] Keep track of datasette.config_dir (#1766) Thanks, @eyeseast - closes #1764 --- datasette/app.py | 1 + tests/test_config_dir.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index edd05bb3..1a9afc10 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -211,6 +211,7 @@ class Datasette: assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" + self.config_dir = config_dir self.pdb = pdb self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 015c6ace..fe927c42 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,4 +1,5 @@ import json +import pathlib import pytest from datasette.app import Datasette @@ -150,3 +151,11 @@ def test_metadata_yaml(tmp_path_factory, filename): response = client.get("/-/metadata.json") assert 200 == response.status assert {"title": "Title from metadata"} == response.json + + +def test_store_config_dir(config_dir_client): + ds = config_dir_client.ds + + assert hasattr(ds, "config_dir") + assert ds.config_dir is not None + assert isinstance(ds.config_dir, pathlib.Path) From 7af67b54b7d9bca43e948510fc62f6db2b748fa8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 18 Jul 2022 14:31:09 -0700 Subject: [PATCH 0737/1866] How to register temporary plugins in tests, closes #903 --- docs/testing_plugins.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 41046bfb..d02003a9 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -219,3 +219,39 @@ Here's a test for that plugin that mocks the HTTPX outbound request: assert ( outbound_request.url == "https://www.example.com/" ) + +.. _testing_plugins_register_in_test: + +Registering a plugin for the duration of a test +----------------------------------------------- + +When writing tests for plugins you may find it useful to register a test plugin just for the duration of a single test. You can do this using ``pm.register()`` and ``pm.unregister()`` like this: + +.. code-block:: python + + from datasette import hookimpl + from datasette.app import Datasette + from datasette.plugins import pm + import pytest + + + @pytest.mark.asyncio + async def test_using_test_plugin(): + class TestPlugin: + __name__ = "TestPlugin" + + # Use hookimpl and method names to register hooks + @hookimpl + def register_routes(self): + return [ + (r"^/error$", lambda: 1/0), + ] + + pm.register(TestPlugin(), name="undo") + try: + # The test implementation goes here + datasette = Datasette() + response = await datasette.client.get("/error") + assert response.status_code == 500 + finally: + pm.unregister(name="undo") From bca2d95d0228f80a108e13408f8e72b2c06c2c7b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 2 Aug 2022 16:38:02 -0700 Subject: [PATCH 0738/1866] Configure readthedocs/readthedocs-preview --- .github/workflows/documentation-links.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/documentation-links.yml diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000..e7062a46 --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request_target: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/readthedocs-preview@main + with: + project-slug: "datasette" From 8cfc72336878dd846d149658e99cc598e835b661 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 9 Aug 2022 11:21:53 -0700 Subject: [PATCH 0739/1866] Ran blacken-docs --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index d02003a9..992b4b0e 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -244,7 +244,7 @@ When writing tests for plugins you may find it useful to register a test plugin @hookimpl def register_routes(self): return [ - (r"^/error$", lambda: 1/0), + (r"^/error$", lambda: 1 / 0), ] pm.register(TestPlugin(), name="undo") From 05d9c682689a0f1d23cbb502e027364ab3363910 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:16:53 -0700 Subject: [PATCH 0740/1866] Promote Discord more in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 032180aa..7ebbca57 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples * Live demo of current main: https://latest.datasette.io/ -* Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) +* Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. From db00c00f6397287749331e8042fe998ee7f3b919 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:19:30 -0700 Subject: [PATCH 0741/1866] Promote Datasette Lite in the README, refs #1781 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ebbca57..1af20129 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples -* Live demo of current main: https://latest.datasette.io/ +* Live demo of current `main` branch: https://latest.datasette.io/ * Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. @@ -85,3 +85,7 @@ Or: This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API. See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details. + +## Datasette Lite + +[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md). From 8eb699de7becdefc6d72555d9fb17c9f06235dc4 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:24:39 -0700 Subject: [PATCH 0742/1866] Datasette Lite in Getting Started docs, closes #1781 --- docs/getting_started.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 00b753a9..571540cf 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -21,6 +21,17 @@ Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you - `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database. - `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data. +.. _getting_started_datasette_lite: + +Datasette in your browser with Datasette Lite +--------------------------------------------- + +`Datasette Lite <https://lite.datasette.io/>`__ is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. + +You can pass a URL to a CSV, SQLite or raw SQL file directly to Datasette Lite to explore that data in your browser. + +This `example link <https://lite.datasette.io/?url=https%3A%2F%2Fraw.githubusercontent.com%2FNUKnightLab%2Fsql-mysteries%2Fmaster%2Fsql-murder-mystery.db#/sql-murder-mystery>`__ opens Datasette Lite and loads the SQL Murder Mystery example database from `Northwestern University Knight Lab <https://github.com/NUKnightLab/sql-mysteries>`__. + .. _getting_started_glitch: Try Datasette without installing anything using Glitch From df4fd2d7ddca8956d8a51c72ce007b8c75227f32 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:44:02 -0700 Subject: [PATCH 0743/1866] _sort= works even if sort column not selected, closes #1773 --- datasette/views/table.py | 22 +++++++++++++++++++++- tests/test_table_api.py | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index cd4be823..94d2673b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -630,7 +630,27 @@ class TableView(DataView): next_value = path_from_row_pks(rows[-2], pks, use_rowid) # If there's a sort or sort_desc, add that value as a prefix if (sort or sort_desc) and not is_view: - prefix = rows[-2][sort or sort_desc] + try: + prefix = rows[-2][sort or sort_desc] + except IndexError: + # sort/sort_desc column missing from SELECT - look up value by PK instead + prefix_where_clause = " and ".join( + "[{}] = :pk{}".format(pk, i) for i, pk in enumerate(pks) + ) + prefix_lookup_sql = "select [{}] from [{}] where {}".format( + sort or sort_desc, table_name, prefix_where_clause + ) + prefix = ( + await db.execute( + prefix_lookup_sql, + { + **{ + "pk{}".format(i): rows[-2][pk] + for i, pk in enumerate(pks) + } + }, + ) + ).single_value() if isinstance(prefix, dict) and "value" in prefix: prefix = prefix["value"] if prefix is None: diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 9db383c3..e56a72b5 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -288,6 +288,8 @@ def test_paginate_compound_keys_with_extra_filters(app_client): ), # text column contains '$null' - ensure it doesn't confuse pagination: ("_sort=text", lambda row: row["text"], "sorted by text"), + # Still works if sort column removed using _col= + ("_sort=text&_col=content", lambda row: row["text"], "sorted by text"), ], ) def test_sortable(app_client, query_string, sort_key, human_description_en): From 668415df9f6334bd255c22ab02018bed5bc14edd Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:47:17 -0700 Subject: [PATCH 0744/1866] Upgrade Docker baes to 3.10.6-slim-bullseye - refs #1768 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42f5529b..ee7ed957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-bullseye as build +FROM python:3.10.6-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 From 080d4b3e065d78faf977c6ded6ead31aae24e2ae Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 08:49:14 -0700 Subject: [PATCH 0745/1866] Switch to python:3.10.6-slim-bullseye for datasette publish - refs #1768 --- datasette/utils/__init__.py | 2 +- demos/apache-proxy/Dockerfile | 2 +- docs/publish.rst | 2 +- tests/test_package.py | 2 +- tests/test_publish_cloudrun.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 77768112..d148cc2c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -390,7 +390,7 @@ def make_dockerfile( "SQLITE_EXTENSIONS" ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" return """ -FROM python:3.8 +FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app {apt_get_extras} diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 6c921963..70b33bec 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-bullseye +FROM python:3.10.6-slim-bullseye RUN apt-get update && \ apt-get install -y apache2 supervisor && \ diff --git a/docs/publish.rst b/docs/publish.rst index 166f2883..9c7c99cc 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -144,7 +144,7 @@ Here's example output for the package command:: $ datasette package parlgov.db --extra-options="--setting sql_time_limit_ms 2500" Sending build context to Docker daemon 4.459MB - Step 1/7 : FROM python:3 + Step 1/7 : FROM python:3.10.6-slim-bullseye ---> 79e1dc9af1c1 Step 2/7 : COPY . /app ---> Using cache diff --git a/tests/test_package.py b/tests/test_package.py index 02ed1775..ac15e61e 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -12,7 +12,7 @@ class CaptureDockerfile: EXPECTED_DOCKERFILE = """ -FROM python:3.8 +FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 3427f4f7..60079ab3 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -223,7 +223,7 @@ def test_publish_cloudrun_plugin_secrets( ) expected = textwrap.dedent( r""" - FROM python:3.8 + FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app @@ -290,7 +290,7 @@ def test_publish_cloudrun_apt_get_install( ) expected = textwrap.dedent( r""" - FROM python:3.8 + FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app From 1563c22a8c65e6cff5194aa07df54d0ab8d4eecb Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 09:13:12 -0700 Subject: [PATCH 0746/1866] Don't duplicate _sort_desc, refs #1738 --- datasette/views/table.py | 2 +- tests/test_table_html.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 94d2673b..49c30c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -710,7 +710,7 @@ class TableView(DataView): for key in request.args: if ( key.startswith("_") - and key not in ("_sort", "_search", "_next") + and key not in ("_sort", "_sort_desc", "_search", "_next") and "__" not in key ): for value in request.args.getlist(key): diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d3cb3e17..f3808ea3 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -828,6 +828,7 @@ def test_other_hidden_form_fields(app_client, path, expected_hidden): [ ("/fixtures/searchable?_search=terry", []), ("/fixtures/searchable?_sort=text2", []), + ("/fixtures/searchable?_sort_desc=text2", []), ("/fixtures/searchable?_sort=text2&_where=1", [("_where", "1")]), ], ) From c1396bf86033a7bd99fa0c0431f585475391a11a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 09:34:31 -0700 Subject: [PATCH 0747/1866] Don't allow canned write queries on immutable DBs, closes #1728 --- datasette/templates/query.html | 6 ++++- datasette/views/database.py | 4 ++++ tests/test_canned_queries.py | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8c920527..cee779fc 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -28,6 +28,10 @@ {% block content %} +{% if canned_write and db_is_immutable %} + <p class="message-error">This query cannot be executed because the database is immutable.</p> +{% endif %} + <h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color(database) }}">{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}</h1> {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} @@ -61,7 +65,7 @@ <p> {% if not hide_sql %}<button id="sql-format" type="button" hidden>Format SQL</button>{% endif %} {% if canned_write %}<input type="hidden" name="csrftoken" value="{{ csrftoken() }}">{% endif %} - <input type="submit" value="Run SQL"> + <input type="submit" value="Run SQL"{% if canned_write and db_is_immutable %} disabled{% endif %}> {{ show_hide_hidden }} {% if canned_query and edit_sql_url %}<a href="{{ edit_sql_url }}" class="canned-query-edit-sql">Edit SQL</a>{% endif %} </p> diff --git a/datasette/views/database.py b/datasette/views/database.py index 42058752..77632b9d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -273,6 +273,9 @@ class QueryView(DataView): # Execute query - as write or as read if write: if request.method == "POST": + # If database is immutable, return an error + if not db.is_mutable: + raise Forbidden("Database is immutable") body = await request.post_body() body = body.decode("utf-8").strip() if body.startswith("{") and body.endswith("}"): @@ -326,6 +329,7 @@ class QueryView(DataView): async def extra_template(): return { "request": request, + "db_is_immutable": not db.is_mutable, "path_with_added_args": path_with_added_args, "path_with_removed_args": path_with_removed_args, "named_parameter_values": named_parameter_values, diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 5abffdcc..976aa0db 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -53,6 +53,26 @@ def canned_write_client(tmpdir): yield client +@pytest.fixture +def canned_write_immutable_client(): + with make_app_client( + is_immutable=True, + metadata={ + "databases": { + "fixtures": { + "queries": { + "add": { + "sql": "insert into sortable (text) values (:text)", + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + def test_canned_query_with_named_parameter(app_client): response = app_client.get("/fixtures/neighborhood_search.json?text=town") assert [ @@ -373,3 +393,23 @@ def test_canned_write_custom_template(canned_write_client): response.headers["link"] == 'http://localhost/data/update_name.json; rel="alternate"; type="application/json+datasette"' ) + + +def test_canned_write_query_disabled_for_immutable_database( + canned_write_immutable_client, +): + response = canned_write_immutable_client.get("/fixtures/add") + assert response.status == 200 + assert ( + "This query cannot be executed because the database is immutable." + in response.text + ) + assert '<input type="submit" value="Run SQL" disabled>' in response.text + # Submitting form should get a forbidden error + response = canned_write_immutable_client.post( + "/fixtures/add", + {"text": "text"}, + csrftoken_from=True, + ) + assert response.status == 403 + assert "Database is immutable" in response.text From 82167105ee699c850cc106ea927de1ad09276cfe Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 10:07:30 -0700 Subject: [PATCH 0748/1866] --min-instances and --max-instances Cloud Run publish options, closes #1779 --- datasette/publish/cloudrun.py | 26 +++++++++++++++++--- docs/cli-reference.rst | 2 ++ tests/test_publish_cloudrun.py | 43 ++++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 50b2b2fd..77274eb0 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -52,6 +52,16 @@ def publish_subcommand(publish): multiple=True, help="Additional packages to apt-get install", ) + @click.option( + "--max-instances", + type=int, + help="Maximum Cloud Run instances", + ) + @click.option( + "--min-instances", + type=int, + help="Minimum Cloud Run instances", + ) def cloudrun( files, metadata, @@ -79,6 +89,8 @@ def publish_subcommand(publish): cpu, timeout, apt_get_extras, + max_instances, + min_instances, ): "Publish databases to Datasette running on Cloud Run" fail_if_publish_binary_not_installed( @@ -168,12 +180,20 @@ def publish_subcommand(publish): ), shell=True, ) + extra_deploy_options = [] + for option, value in ( + ("--memory", memory), + ("--cpu", cpu), + ("--max-instances", max_instances), + ("--min-instances", min_instances), + ): + if value: + extra_deploy_options.append("{} {}".format(option, value)) check_call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, service, - " --memory {}".format(memory) if memory else "", - " --cpu {}".format(cpu) if cpu else "", + " " + " ".join(extra_deploy_options) if extra_deploy_options else "", ), shell=True, ) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 1c1aff15..415af13c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -251,6 +251,8 @@ datasette publish cloudrun --help --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run --timeout INTEGER Build timeout in seconds --apt-get-install TEXT Additional packages to apt-get install + --max-instances INTEGER Maximum Cloud Run instances + --min-instances INTEGER Minimum Cloud Run instances --help Show this message and exit. diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 60079ab3..e64534d2 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -105,19 +105,36 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @pytest.mark.parametrize( - "memory,cpu,timeout,expected_gcloud_args", + "memory,cpu,timeout,min_instances,max_instances,expected_gcloud_args", [ - ["1Gi", None, None, "--memory 1Gi"], - ["2G", None, None, "--memory 2G"], - ["256Mi", None, None, "--memory 256Mi"], - ["4", None, None, None], - ["GB", None, None, None], - [None, 1, None, "--cpu 1"], - [None, 2, None, "--cpu 2"], - [None, 3, None, None], - [None, 4, None, "--cpu 4"], - ["2G", 4, None, "--memory 2G --cpu 4"], - [None, None, 1800, "--timeout 1800"], + ["1Gi", None, None, None, None, "--memory 1Gi"], + ["2G", None, None, None, None, "--memory 2G"], + ["256Mi", None, None, None, None, "--memory 256Mi"], + [ + "4", + None, + None, + None, + None, + None, + ], + [ + "GB", + None, + None, + None, + None, + None, + ], + [None, 1, None, None, None, "--cpu 1"], + [None, 2, None, None, None, "--cpu 2"], + [None, 3, None, None, None, None], + [None, 4, None, None, None, "--cpu 4"], + ["2G", 4, None, None, None, "--memory 2G --cpu 4"], + [None, None, 1800, None, None, "--timeout 1800"], + [None, None, None, 2, None, "--min-instances 2"], + [None, None, None, 2, 4, "--min-instances 2 --max-instances 4"], + [None, 2, None, None, 4, "--cpu 2 --max-instances 4"], ], ) def test_publish_cloudrun_memory_cpu( @@ -127,6 +144,8 @@ def test_publish_cloudrun_memory_cpu( memory, cpu, timeout, + min_instances, + max_instances, expected_gcloud_args, tmp_path_factory, ): From 5e6c5c9e3191a80f17a91c5205d9d69efdebb73f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 10:18:47 -0700 Subject: [PATCH 0749/1866] Document datasette.config_dir, refs #1766 --- docs/internals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/internals.rst b/docs/internals.rst index da135282..20797e98 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -260,6 +260,7 @@ Constructor parameters include: - ``files=[...]`` - a list of database files to open - ``immutables=[...]`` - a list of database files to open in immutable mode - ``metadata={...}`` - a dictionary of :ref:`metadata` +- ``config_dir=...`` - the :ref:`configuration directory <config_dir>` to use, stored in ``datasette.config_dir`` .. _datasette_databases: From 815162cf029fab9f1c9308c1d6ecdba7ee369ebe Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 10:32:42 -0700 Subject: [PATCH 0750/1866] Release 0.62 Refs #903, #1300, #1683, #1701, #1712, #1717, #1718, #1728, #1733, #1738, #1739, #1744, #1746, #1748, #1759, #1766, #1768, #1770, #1773, #1779 Closes #1782 --- datasette/version.py | 2 +- docs/changelog.rst | 53 ++++++++++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 86f4cf7e..0453346c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62a1" +__version__ = "0.62" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f105811..1225c63f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,33 +4,52 @@ Changelog ========= -.. _v0_62a1: +.. _v0_62: -0.62a1 (2022-07-17) +0.62 (2022-08-14) ------------------- +Datasette can now run entirely in your browser using WebAssembly. Try out `Datasette Lite <https://lite.datasette.io/>`__, take a look `at the code <https://github.com/simonw/datasette-lite>`__ or read more about it in `Datasette Lite: a server-side Python web application running in a browser <https://simonwillison.net/2022/May/4/datasette-lite/>`__. + +Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__ for questions and discussions about Datasette and its ecosystem of projects. + +Features +~~~~~~~~ + +- Datasette is now compatible with `Pyodide <https://pyodide.org/>`__. This is the enabling technology behind `Datasette Lite <https://lite.datasette.io/>`__. (:issue:`1733`) +- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) +- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__) +- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details. +- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) +- Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) +- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) +- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``. (:issue:`1768`) +- Canned writable queries against immutable databases now show a warning message. (:issue:`1728`) +- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__) +- ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`) + +Plugin hooks +~~~~~~~~~~~~ + - New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`) - The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`) -- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) -- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`) -- Datasette now has a `Discord community <https://discord.gg/ktd74dm5mw>`__. -- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) -- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) -- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__) +- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (`#1766 <https://github.com/simonw/datasette/pull/1766>`__) -.. _v0_62a0: +Bug fixes +~~~~~~~~~ -0.62a0 (2022-05-02) -------------------- - -- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details. -- Datasette should now be compatible with Pyodide. (:issue:`1733`) -- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__) -- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) -- ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) - Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`) +- ``?_sort`` and ``?_sort_desc`` now work if the column that is being sorted has been excluded from the query using ``?_col=`` or ``?_nocol=``. (:issue:`1773`) +- Fixed bug where ``?_sort_desc`` was duplicated in the URL every time the Apply button was clicked. (:issue:`1738`) + +Documentation +~~~~~~~~~~~~~ + +- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) +- Documentation now uses the `Furo <https://github.com/pradyunsg/furo>`__ Sphinx theme. (:issue:`1746`) - Code examples in the documentation are now all formatted using Black. (:issue:`1718`) - ``Request.fake()`` method is now documented, see :ref:`internals_request`. +- New documentation for plugin authors: :ref:`testing_plugins_register_in_test`. (:issue:`903`) .. _v0_61_1: From a107e3a028923c1ab3911c0f880011283f93f368 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 14 Aug 2022 16:07:46 -0700 Subject: [PATCH 0751/1866] datasette-sentry is an example of handle_exception --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index aec1df56..c6f35d06 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1261,6 +1261,8 @@ This example logs an error to `Sentry <https://sentry.io/>`__ and then renders a return inner +Example: `datasette-sentry <https://datasette.io/plugins/datasette-sentry>`_ + .. _plugin_hook_menu_links: menu_links(datasette, actor, request) From 481eb96d85291cdfa5767a83884a1525dfc382d8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 15 Aug 2022 13:17:28 -0700 Subject: [PATCH 0752/1866] https://datasette.io/tutorials/clean-data tutorial Refs #1783 --- docs/getting_started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 571540cf..a9eaa404 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -20,6 +20,7 @@ Datasette has several `tutorials <https://datasette.io/tutorials>`__ to help you - `Exploring a database with Datasette <https://datasette.io/tutorials/explore>`__ shows how to use the Datasette web interface to explore a new database. - `Learn SQL with Datasette <https://datasette.io/tutorials/learn-sql>`__ introduces SQL, and shows how to use that query language to ask questions of your data. +- `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__ guides you through using `sqlite-utils <https://sqlite-utils.datasette.io/>`__ to turn a CSV file into a database that you can explore using Datasette. .. _getting_started_datasette_lite: From a3e6f1b16757fb2d39e7ddba4e09eda2362508bf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 18 Aug 2022 09:06:02 -0700 Subject: [PATCH 0753/1866] Increase height of non-JS textarea to fit query Closes #1786 --- datasette/templates/query.html | 3 ++- tests/test_html.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index cee779fc..a35e3afe 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -45,7 +45,8 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} - <p><textarea id="sql-editor" name="sql">{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p> + <p><textarea id="sql-editor" name="sql"{% if query and query.sql %} style="height: {{ query.sql.split("\n")|length + 2 }}em"{% endif %} + >{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p> {% else %} <pre id="sql-query">{% if query %}{{ query.sql }}{% endif %}</pre> {% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 409fec68..be21bd84 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -695,10 +695,8 @@ def test_query_error(app_client): response = app_client.get("/fixtures?sql=select+*+from+notatable") html = response.text assert '<p class="message-error">no such table: notatable</p>' in html - assert ( - '<textarea id="sql-editor" name="sql">select * from notatable</textarea>' - in html - ) + assert '<textarea id="sql-editor" name="sql" style="height: 3em' in html + assert ">select * from notatable</textarea>" in html assert "0 results" not in html From 09a41662e70b788469157bb58ed9ca4acdf2f904 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 18 Aug 2022 09:10:48 -0700 Subject: [PATCH 0754/1866] Fix typo --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c6f35d06..30bd75b7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -874,7 +874,7 @@ canned_queries(datasette, database, actor) ``actor`` - dictionary or None The currently authenticated :ref:`actor <authentication_actor>`. -Ues this hook to return a dictionary of additional :ref:`canned query <canned_queries>` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query <canned_queries>` documentation. +Use this hook to return a dictionary of additional :ref:`canned query <canned_queries>` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query <canned_queries>` documentation. .. code-block:: python From 6c0ba7c00c2ae3ecbb5309efa59079cea1c850b3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 18 Aug 2022 14:52:04 -0700 Subject: [PATCH 0755/1866] Improved CLI reference documentation, refs #1787 --- datasette/cli.py | 2 +- docs/changelog.rst | 2 +- docs/cli-reference.rst | 325 ++++++++++++++++++++++++++++++--------- docs/getting_started.rst | 50 ------ docs/index.rst | 2 +- docs/publish.rst | 2 + 6 files changed, 259 insertions(+), 124 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 8781747c..f2a03d53 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -282,7 +282,7 @@ def package( port, **extra_metadata, ): - """Package specified SQLite files into a new datasette Docker container""" + """Package SQLite files into a Datasette Docker container""" if not shutil.which("docker"): click.secho( ' The package command requires "docker" to be installed and configured ', diff --git a/docs/changelog.rst b/docs/changelog.rst index 1225c63f..f9dcc980 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -621,7 +621,7 @@ See also `Datasette 0.49: The annotated release notes <https://simonwillison.net - Datasette now has `a GitHub discussions forum <https://github.com/simonw/datasette/discussions>`__ for conversations about the project that go beyond just bug reports and issues. - Datasette can now be installed on macOS using Homebrew! Run ``brew install simonw/datasette/datasette``. See :ref:`installation_homebrew`. (:issue:`335`) - Two new commands: ``datasette install name-of-plugin`` and ``datasette uninstall name-of-plugin``. These are equivalent to ``pip install`` and ``pip uninstall`` but automatically run in the same virtual environment as Datasette, so users don't have to figure out where that virtual environment is - useful for installations created using Homebrew or ``pipx``. See :ref:`plugins_installing`. (:issue:`925`) -- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`) +- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`cli_datasette_get` for an example. (:issue:`926`) .. _v0_46: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 415af13c..a1e56774 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -4,44 +4,34 @@ CLI reference =============== -This page lists the ``--help`` for every ``datasette`` CLI command. +The ``datasette`` CLI tool provides a number of commands. + +Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. .. [[[cog from datasette import cli from click.testing import CliRunner import textwrap - commands = [ - ["--help"], - ["serve", "--help"], - ["serve", "--help-settings"], - ["plugins", "--help"], - ["publish", "--help"], - ["publish", "cloudrun", "--help"], - ["publish", "heroku", "--help"], - ["package", "--help"], - ["inspect", "--help"], - ["install", "--help"], - ["uninstall", "--help"], - ] - cog.out("\n") - for command in commands: - title = "datasette " + " ".join(command) - ref = "_cli_help_" + ("_".join(command).replace("-", "_")) - cog.out(".. {}:\n\n".format(ref)) - cog.out(title + "\n") - cog.out(("=" * len(title)) + "\n\n") + def help(args): + title = "datasette " + " ".join(args) cog.out("::\n\n") - result = CliRunner().invoke(cli.cli, command) + result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) cog.out("\n\n") .. ]]] +.. [[[end]]] .. _cli_help___help: datasette --help ================ +Running ``datasette --help`` shows a list of all of the available commands. + +.. [[[cog + help(["--help"]) +.. ]]] :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -59,17 +49,34 @@ datasette --help serve* Serve up specified SQLite database files with a web UI inspect Generate JSON summary of provided database files install Install plugins and packages from PyPI into the same... - package Package specified SQLite files into a new datasette Docker... + package Package SQLite files into a Datasette Docker container plugins List currently installed plugins publish Publish specified SQLite database files to the internet along... uninstall Uninstall plugins and Python packages from the Datasette... +.. [[[end]]] + +Additional commands added by plugins that use the :ref:`plugin_hook_register_commands` hook will be listed here as well. + .. _cli_help_serve___help: -datasette serve --help -====================== +datasette serve +=============== +This command starts the Datasette web application running on your machine:: + + datasette serve mydatabase.db + +Or since this is the default command you can run this instead:: + + datasette mydatabase.db + +Once started you can access it at ``http://localhost:8001`` + +.. [[[cog + help(["serve", "--help"]) +.. ]]] :: Usage: datasette serve [OPTIONS] [FILES]... @@ -121,11 +128,75 @@ datasette serve --help --help Show this message and exit. +.. [[[end]]] + + +.. _cli_datasette_get: + +datasette --get +--------------- + +The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies the path to a page within Datasette and causes Datasette to output the content from that path without starting the web server. + +This means that all of Datasette's functionality can be accessed directly from the command-line. + +For example:: + + $ datasette --get '/-/versions.json' | jq . + { + "python": { + "version": "3.8.5", + "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" + }, + "datasette": { + "version": "0.46+15.g222a84a.dirty" + }, + "asgi": "3.0", + "uvicorn": "0.11.8", + "sqlite": { + "version": "3.32.3", + "fts_versions": [ + "FTS5", + "FTS4", + "FTS3" + ], + "extensions": { + "json1": null + }, + "compile_options": [ + "COMPILER=clang-11.0.3", + "ENABLE_COLUMN_METADATA", + "ENABLE_FTS3", + "ENABLE_FTS3_PARENTHESIS", + "ENABLE_FTS4", + "ENABLE_FTS5", + "ENABLE_GEOPOLY", + "ENABLE_JSON1", + "ENABLE_PREUPDATE_HOOK", + "ENABLE_RTREE", + "ENABLE_SESSION", + "MAX_VARIABLE_NUMBER=250000", + "THREADSAFE=1" + ] + } + } + +The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. + +This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. + .. _cli_help_serve___help_settings: datasette serve --help-settings -=============================== +------------------------------- +This command outputs all of the available Datasette :ref:`settings <settings>`. + +These can be passed to ``datasette serve`` using ``datasette serve --setting name value``. + +.. [[[cog + help(["--help-settings"]) +.. ]]] :: Settings: @@ -170,11 +241,18 @@ datasette serve --help-settings +.. [[[end]]] + .. _cli_help_plugins___help: -datasette plugins --help -======================== +datasette plugins +================= +Output JSON showing all currently installed plugins, their versions, whether they include static files or templates and which :ref:`plugin_hooks` they use. + +.. [[[cog + help(["plugins", "--help"]) +.. ]]] :: Usage: datasette plugins [OPTIONS] @@ -187,11 +265,110 @@ datasette plugins --help --help Show this message and exit. +.. [[[end]]] + +Example output: + +.. code-block:: json + + [ + { + "name": "datasette-geojson", + "static": false, + "templates": false, + "version": "0.3.1", + "hooks": [ + "register_output_renderer" + ] + }, + { + "name": "datasette-geojson-map", + "static": true, + "templates": false, + "version": "0.4.0", + "hooks": [ + "extra_body_script", + "extra_css_urls", + "extra_js_urls" + ] + }, + { + "name": "datasette-leaflet", + "static": true, + "templates": false, + "version": "0.2.2", + "hooks": [ + "extra_body_script", + "extra_template_vars" + ] + } + ] + + +.. _cli_help_install___help: + +datasette install +================= + +Install new Datasette plugins. This command works like ``pip install`` but ensures that your plugins will be installed into the same environment as Datasette. + +This command:: + + datasette install datasette-cluster-map + +Would install the `datasette-cluster-map <https://datasette.io/plugins/datasette-cluster-map>`__ plugin. + +.. [[[cog + help(["install", "--help"]) +.. ]]] +:: + + Usage: datasette install [OPTIONS] PACKAGES... + + Install plugins and packages from PyPI into the same environment as Datasette + + Options: + -U, --upgrade Upgrade packages to latest version + --help Show this message and exit. + + +.. [[[end]]] + +.. _cli_help_uninstall___help: + +datasette uninstall +=================== + +Uninstall one or more plugins. + +.. [[[cog + help(["uninstall", "--help"]) +.. ]]] +:: + + Usage: datasette uninstall [OPTIONS] PACKAGES... + + Uninstall plugins and Python packages from the Datasette environment + + Options: + -y, --yes Don't ask for confirmation + --help Show this message and exit. + + +.. [[[end]]] + .. _cli_help_publish___help: -datasette publish --help -======================== +datasette publish +================= +Shows a list of available deployment targets for :ref:`publishing data <publishing>` with Datasette. + +Additional deployment targets can be added by plugins that use the :ref:`plugin_hook_publish_subcommand` hook. + +.. [[[cog + help(["publish", "--help"]) +.. ]]] :: Usage: datasette publish [OPTIONS] COMMAND [ARGS]... @@ -207,11 +384,19 @@ datasette publish --help heroku Publish databases to Datasette running on Heroku +.. [[[end]]] + + .. _cli_help_publish_cloudrun___help: -datasette publish cloudrun --help -================================= +datasette publish cloudrun +========================== +See :ref:`publish_cloud_run`. + +.. [[[cog + help(["publish", "cloudrun", "--help"]) +.. ]]] :: Usage: datasette publish cloudrun [OPTIONS] [FILES]... @@ -256,11 +441,19 @@ datasette publish cloudrun --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_publish_heroku___help: -datasette publish heroku --help -=============================== +datasette publish heroku +======================== +See :ref:`publish_heroku`. + +.. [[[cog + help(["publish", "heroku", "--help"]) +.. ]]] :: Usage: datasette publish heroku [OPTIONS] [FILES]... @@ -297,16 +490,23 @@ datasette publish heroku --help --help Show this message and exit. +.. [[[end]]] + .. _cli_help_package___help: -datasette package --help -======================== +datasette package +================= +Package SQLite files into a Datasette Docker container, see :ref:`cli_package`. + +.. [[[cog + help(["package", "--help"]) +.. ]]] :: Usage: datasette package [OPTIONS] FILES... - Package specified SQLite files into a new datasette Docker container + Package SQLite files into a Datasette Docker container Options: -t, --tag TEXT Name for the resulting Docker container, can @@ -335,11 +535,26 @@ datasette package --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_inspect___help: -datasette inspect --help -======================== +datasette inspect +================= +Outputs JSON representing introspected data about one or more SQLite database files. + +If you are opening an immutable database, you can pass this file to the ``--inspect-data`` option to improve Datasette's performance by allowing it to skip running row counts against the database when it first starts running:: + + datasette inspect mydatabase.db > inspect-data.json + datasette serve -i mydatabase.db --inspect-file inspect-data.json + +This performance optimization is used automatically by some of the ``datasette publish`` commands. You are unlikely to need to apply this optimization manually. + +.. [[[cog + help(["inspect", "--help"]) +.. ]]] :: Usage: datasette inspect [OPTIONS] [FILES]... @@ -355,36 +570,4 @@ datasette inspect --help --help Show this message and exit. -.. _cli_help_install___help: - -datasette install --help -======================== - -:: - - Usage: datasette install [OPTIONS] PACKAGES... - - Install plugins and packages from PyPI into the same environment as Datasette - - Options: - -U, --upgrade Upgrade packages to latest version - --help Show this message and exit. - - -.. _cli_help_uninstall___help: - -datasette uninstall --help -========================== - -:: - - Usage: datasette uninstall [OPTIONS] PACKAGES... - - Uninstall plugins and Python packages from the Datasette environment - - Options: - -y, --yes Don't ask for confirmation - --help Show this message and exit. - - .. [[[end]]] diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a9eaa404..6515ef8d 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -138,53 +138,3 @@ JSON in a more convenient format: } ] } - -.. _getting_started_datasette_get: - -datasette --get ---------------- - -The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example:: - - $ datasette --get '/-/versions.json' | jq . - { - "python": { - "version": "3.8.5", - "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" - }, - "datasette": { - "version": "0.46+15.g222a84a.dirty" - }, - "asgi": "3.0", - "uvicorn": "0.11.8", - "sqlite": { - "version": "3.32.3", - "fts_versions": [ - "FTS5", - "FTS4", - "FTS3" - ], - "extensions": { - "json1": null - }, - "compile_options": [ - "COMPILER=clang-11.0.3", - "ENABLE_COLUMN_METADATA", - "ENABLE_FTS3", - "ENABLE_FTS3_PARENTHESIS", - "ENABLE_FTS4", - "ENABLE_FTS5", - "ENABLE_GEOPOLY", - "ENABLE_JSON1", - "ENABLE_PREUPDATE_HOOK", - "ENABLE_RTREE", - "ENABLE_SESSION", - "MAX_VARIABLE_NUMBER=250000", - "THREADSAFE=1" - ] - } - } - -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. - -Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. diff --git a/docs/index.rst b/docs/index.rst index efe196b3..5a9cc7ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ Contents getting_started installation ecosystem + cli-reference pages publish deploying @@ -61,6 +62,5 @@ Contents plugin_hooks testing_plugins internals - cli-reference contributing changelog diff --git a/docs/publish.rst b/docs/publish.rst index 9c7c99cc..dd8566ed 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -56,6 +56,8 @@ Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command. +.. _publish_heroku: + Publishing to Heroku -------------------- From aff3df03d4fe0806ce432d1818f6643cdb2a854e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 18 Aug 2022 14:55:08 -0700 Subject: [PATCH 0756/1866] Ignore ro which stands for read only Refs #1787 where it caused tests to break --- docs/codespell-ignore-words.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index a625cde5..d6744d05 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -1 +1 @@ -AddWordsToIgnoreHere +ro From 0d9d33955b503c88a2c712144d97f094baa5d46d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 18 Aug 2022 16:06:12 -0700 Subject: [PATCH 0757/1866] Clarify you can publish multiple files, closes #1788 --- docs/publish.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index dd8566ed..d817ed31 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -31,7 +31,7 @@ Publishing to Google Cloud Run You will first need to install and configure the Google Cloud CLI tools by following `these instructions <https://cloud.google.com/sdk/>`__. -You can then publish a database to Google Cloud Run using the following command:: +You can then publish one or more SQLite database files to Google Cloud Run using the following command:: datasette publish cloudrun mydatabase.db --service=my-database @@ -63,7 +63,7 @@ Publishing to Heroku To publish your data using `Heroku <https://www.heroku.com/>`__, first create an account there and install and configure the `Heroku CLI tool <https://devcenter.heroku.com/articles/heroku-cli>`_. -You can publish a database to Heroku using the following command:: +You can publish one or more databases to Heroku using the following command:: datasette publish heroku mydatabase.db @@ -138,7 +138,7 @@ If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plug datasette package ================= -If you have docker installed (e.g. using `Docker for Mac <https://www.docker.com/docker-mac>`_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with your selected SQLite databases:: +If you have docker installed (e.g. using `Docker for Mac <https://www.docker.com/docker-mac>`_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with one or more SQLite databases:: datasette package mydatabase.db From 663ac431fe7202c85967568d82b2034f92b9aa43 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann <humitos@gmail.com> Date: Sat, 20 Aug 2022 02:04:16 +0200 Subject: [PATCH 0758/1866] Use Read the Docs action v1 (#1778) Read the Docs repository was renamed from `readthedocs/readthedocs-preview` to `readthedocs/actions/`. Now, the `preview` action is under `readthedocs/actions/preview` and is tagged as `v1` --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index e7062a46..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -11,6 +11,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/readthedocs-preview@main + - uses: readthedocs/actions/preview@v1 with: project-slug: "datasette" From 1d64c9a8dac45b9a3452acf8e76dfadea2b0bc49 Mon Sep 17 00:00:00 2001 From: Alex Garcia <alexsebastian.garcia@gmail.com> Date: Tue, 23 Aug 2022 11:34:30 -0700 Subject: [PATCH 0759/1866] Add new entrypoint option to --load-extensions. (#1789) Thanks, @asg017 --- .gitignore | 6 ++++ datasette/app.py | 8 ++++- datasette/cli.py | 4 ++- datasette/utils/__init__.py | 11 ++++++ tests/ext.c | 48 ++++++++++++++++++++++++++ tests/test_load_extensions.py | 65 +++++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/ext.c create mode 100644 tests/test_load_extensions.py diff --git a/.gitignore b/.gitignore index 066009f0..277ff653 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,9 @@ ENV/ .DS_Store node_modules .*.swp + +# In case someone compiled tests/ext.c for test_load_extensions, don't +# include it in source control. +tests/*.dylib +tests/*.so +tests/*.dll \ No newline at end of file diff --git a/datasette/app.py b/datasette/app.py index 1a9afc10..bb9232c9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,7 +559,13 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - conn.execute("SELECT load_extension(?)", [extension]) + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member diff --git a/datasette/cli.py b/datasette/cli.py index f2a03d53..6eb42712 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( pm, ) from .utils import ( + LoadExtension, StartupError, check_connection, find_spatialite, @@ -128,9 +129,10 @@ def sqlite_extensions(fn): return click.option( "sqlite_extensions", "--load-extension", + type=LoadExtension(), envvar="SQLITE_EXTENSIONS", multiple=True, - help="Path to a SQLite extension to load", + help="Path to a SQLite extension to load, and optional entrypoint", )(fn) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d148cc2c..0fc87d51 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,17 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath +# The --load-extension parameter can optionally include a specific entrypoint. +# This is done by appending ":entrypoint_name" after supplying the path to the extension +class LoadExtension(click.ParamType): + name = "path:entrypoint?" + + def convert(self, value, param, ctx): + if ":" not in value: + return value + path, entrypoint = value.split(":", 1) + return path, entrypoint + def format_bytes(bytes): current = float(bytes) diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 00000000..5fe970d9 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py new file mode 100644 index 00000000..360bc8f3 --- /dev/null +++ b/tests/test_load_extensions.py @@ -0,0 +1,65 @@ +from datasette.app import Datasette +import pytest +from pathlib import Path + +# not necessarily a full path - the full compiled path looks like "ext.dylib" +# or another suffix, but sqlite will, under the hood, decide which file +# extension to use based on the operating system (apple=dylib, windows=dll etc) +# this resolves to "./ext", which is enough for SQLite to calculate the rest +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") + +# See if ext.c has been compiled, based off the different possible suffixes. +def has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_default_entrypoint(): + + # The default entrypoint only loads a() and NOT b() or c(), so those + # should fail. + ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH]) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: c" + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_multiple_entrypoints(): + + # Load in the default entrypoint and the other 2 custom entrypoints, now + # all a(), b(), and c() should run successfully. + ds = Datasette( + sqlite_extensions=[ + COMPILED_EXTENSION_PATH, + (COMPILED_EXTENSION_PATH, "sqlite3_ext_b_init"), + (COMPILED_EXTENSION_PATH, "sqlite3_ext_c_init"), + ] + ) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "c" From fd1086c6867f3e3582b1eca456e4ea95f6cecf8b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 23 Aug 2022 11:35:41 -0700 Subject: [PATCH 0760/1866] Applied Black, refs #1789 --- datasette/app.py | 4 ++-- datasette/utils/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bb9232c9..f2a6763a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,8 +559,8 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. if isinstance(extension, tuple): path, entrypoint = extension conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0fc87d51..bbaa0510 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,7 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath + # The --load-extension parameter can optionally include a specific entrypoint. # This is done by appending ":entrypoint_name" after supplying the path to the extension class LoadExtension(click.ParamType): From 456dc155d491a009942ace71a4e1827cddc6b93d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 23 Aug 2022 11:40:36 -0700 Subject: [PATCH 0761/1866] Ran cog, refs #1789 --- docs/cli-reference.rst | 95 +++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a1e56774..f8419d58 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -84,48 +84,53 @@ Once started you can access it at ``http://localhost:8001`` Serve up specified SQLite database files with a web UI Options: - -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means - only connections from the local machine will be - allowed. Use 0.0.0.0 to listen to all IPs and allow - access from other machines. - -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to - automatically assign an available port. - [0<=x<=65535] - --uds TEXT Bind to a Unix domain socket - --reload Automatically reload if code or metadata change - detected - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: - * - --load-extension TEXT Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON/YAML file containing license/source - metadata - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html - --secret TEXT Secret used for signing secure values, such as - signed cookies - --root Output URL that sets a cookie authenticating the - root user - --get TEXT Run an HTTP GET request against this path, print - results and exit - --version-note TEXT Additional note to show on /-/versions - --help-settings Show available settings - --pdb Launch debugger on any errors - -o, --open Open Datasette in your web browser - --create Create database files if they do not exist - --crossdb Enable cross-database joins using the /_memory - database - --nolock Ignore locking, open locked files in read-only mode - --ssl-keyfile TEXT SSL key file - --ssl-certfile TEXT SSL certificate file - --help Show this message and exit. + -i, --immutable PATH Database files to open in immutable mode + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which + means only connections from the local machine + will be allowed. Use 0.0.0.0 to listen to all + IPs and allow access from other machines. + -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to + automatically assign an available port. + [0<=x<=65535] + --uds TEXT Bind to a Unix domain socket + --reload Automatically reload if code or metadata + change detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON/YAML file containing + license/source metadata + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at + /MOUNT/... + --memory Make /_memory database available + --config CONFIG Deprecated: set config option using + configname:value. Use --setting instead. + --setting SETTING... Setting, see + docs.datasette.io/en/stable/settings.html + --secret TEXT Secret used for signing secure values, such as + signed cookies + --root Output URL that sets a cookie authenticating + the root user + --get TEXT Run an HTTP GET request against this path, + print results and exit + --version-note TEXT Additional note to show on /-/versions + --help-settings Show available settings + --pdb Launch debugger on any errors + -o, --open Open Datasette in your web browser + --create Create database files if they do not exist + --crossdb Enable cross-database joins using the /_memory + database + --nolock Ignore locking, open locked files in read-only + mode + --ssl-keyfile TEXT SSL key file + --ssl-certfile TEXT SSL certificate file + --help Show this message and exit. .. [[[end]]] @@ -566,8 +571,10 @@ This performance optimization is used automatically by some of the ``datasette p Options: --inspect-file TEXT - --load-extension TEXT Path to a SQLite extension to load - --help Show this message and exit. + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --help Show this message and exit. .. [[[end]]] From ba35105eee2d3ba620e4f230028a02b2e2571df2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 23 Aug 2022 17:11:45 -0700 Subject: [PATCH 0762/1866] Test `--load-extension` in GitHub Actions (#1792) * Run the --load-extension test, refs #1789 * Ran cog, refs #1789 --- .github/workflows/test.yml | 3 +++ tests/test_api.py | 2 +- tests/test_html.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90b6555e..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | pip install -e '.[test]' diff --git a/tests/test_api.py b/tests/test_api.py index 253c1718..f6db2f9d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,7 +36,7 @@ def test_homepage(app_client): # 4 hidden FTS tables + no_primary_key (hidden in metadata) assert d["hidden_tables_count"] == 6 # 201 in no_primary_key, plus 6 in other hidden tables: - assert d["hidden_table_rows_sum"] == 207 + assert d["hidden_table_rows_sum"] == 207, response.json assert d["views_count"] == 4 diff --git a/tests/test_html.py b/tests/test_html.py index be21bd84..d6e969ad 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -115,7 +115,7 @@ def test_database_page(app_client): assert fragment in response.text # And views - views_ul = soup.find("h2", text="Views").find_next_sibling("ul") + views_ul = soup.find("h2", string="Views").find_next_sibling("ul") assert views_ul is not None assert [ ("/fixtures/paginated_view", "paginated_view"), @@ -128,7 +128,7 @@ def test_database_page(app_client): ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) # And a list of canned queries - queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") + queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ ("/fixtures/from_async_hook", "from_async_hook"), From 51030df1869b3b574dd3584d1563415776b9cd4e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 11:35:40 -0700 Subject: [PATCH 0763/1866] Don't use upper bound dependencies any more See https://iscinumpy.dev/post/bound-version-constraints/ for the rationale behind this change. Closes #1800 --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index a1c51d0b..b2e50b38 100644 --- a/setup.py +++ b/setup.py @@ -42,21 +42,21 @@ setup( include_package_data=True, python_requires=">=3.7", install_requires=[ - "asgiref>=3.2.10,<3.6.0", - "click>=7.1.1,<8.2.0", + "asgiref>=3.2.10", + "click>=7.1.1", "click-default-group-wheel>=1.2.2", - "Jinja2>=2.10.3,<3.1.0", - "hupper~=1.9", + "Jinja2>=2.10.3", + "hupper>=1.9", "httpx>=0.20", - "pint~=0.9", - "pluggy>=1.0,<1.1", - "uvicorn~=0.11", - "aiofiles>=0.4,<0.9", - "janus>=0.6.2,<1.1", + "pint>=0.9", + "pluggy>=1.0", + "uvicorn>=0.11", + "aiofiles>=0.4", + "janus>=0.6.2", "asgi-csrf>=0.9", - "PyYAML>=5.3,<7.0", - "mergedeep>=1.1.1,<1.4.0", - "itsdangerous>=1.1,<3.0", + "PyYAML>=5.3", + "mergedeep>=1.1.1", + "itsdangerous>=1.1", ], entry_points=""" [console_scripts] @@ -72,14 +72,14 @@ setup( "sphinx-copybutton", ], "test": [ - "pytest>=5.2.2,<7.2.0", - "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.17,<0.20", - "beautifulsoup4>=4.8.1,<4.12.0", + "pytest>=5.2.2", + "pytest-xdist>=2.2.1", + "pytest-asyncio>=0.17", + "beautifulsoup4>=4.8.1", "black==22.6.0", "blacken-docs==1.12.1", - "pytest-timeout>=1.4.2,<2.2", - "trustme>=0.7,<0.10", + "pytest-timeout>=1.4.2", + "trustme>=0.7", "cogapp>=3.3.0", ], "rich": ["rich"], From 294ecd45f7801971dbeef383d0c5456ee95ab839 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 11:51:51 -0700 Subject: [PATCH 0764/1866] Bump black from 22.6.0 to 22.8.0 (#1797) Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2e50b38..92fa60d0 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.6.0", + "black==22.8.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From b91e17280c05bbb9cf97432081bdcea8665879f9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 16:50:53 -0700 Subject: [PATCH 0765/1866] Run tests in serial, refs #1802 --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e38d5ee9..9c8c48ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,8 +33,7 @@ jobs: pip freeze - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + pytest - name: Check if cog needs to be run run: | cog --check docs/*.rst From b2b901e8c4b939e50ee1117ffcd2881ed8a8e3bf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 17:05:23 -0700 Subject: [PATCH 0766/1866] Skip SpatiaLite test if no conn.enable_load_extension() Ran into this problem while working on #1802 --- tests/test_spatialite.py | 2 ++ tests/utils.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py index 8b98c5d6..c07a30e8 100644 --- a/tests/test_spatialite.py +++ b/tests/test_spatialite.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS +from .utils import has_load_extension import pytest @@ -13,6 +14,7 @@ def has_spatialite(): @pytest.mark.asyncio @pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite") +@pytest.mark.skipif(not has_load_extension(), reason="Requires enable_load_extension") async def test_spatialite_version_info(): ds = Datasette(sqlite_extensions=["spatialite"]) response = await ds.client.get("/-/versions.json") diff --git a/tests/utils.py b/tests/utils.py index 972300db..191ead9b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,6 @@ +from datasette.utils.sqlite import sqlite3 + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) @@ -22,3 +25,8 @@ def inner_html(soup): # This includes the parent tag - so remove that inner_html = html.split(">", 1)[1].rsplit("<", 1)[0] return inner_html.strip() + + +def has_load_extension(): + conn = sqlite3.connect(":memory:") + return hasattr(conn, "enable_load_extension") From 1c29b925d300d1ee17047504473f2517767aa05b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 17:10:52 -0700 Subject: [PATCH 0767/1866] Run tests in serial again Because this didn't fix the issue I'm seeing in #1802 Revert "Run tests in serial, refs #1802" This reverts commit b91e17280c05bbb9cf97432081bdcea8665879f9. --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c8c48ef..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,8 @@ jobs: pip freeze - name: Run tests run: | - pytest + pytest -n auto -m "not serial" + pytest -m "serial" - name: Check if cog needs to be run run: | cog --check docs/*.rst From 64288d827f7ff97f825e10f714da3f781ecf9345 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 17:40:19 -0700 Subject: [PATCH 0768/1866] Workaround for test failure: RuntimeError: There is no current event loop (#1803) * Remove ensure_eventloop hack * Hack to recover from intermittent RuntimeError calling asyncio.Lock() --- datasette/app.py | 10 +++++++++- tests/test_cli.py | 27 ++++++++++----------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f2a6763a..c6bbdaf0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -231,7 +231,15 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self._refresh_schemas_lock = asyncio.Lock() + try: + self._refresh_schemas_lock = asyncio.Lock() + except RuntimeError as rex: + # Workaround for intermittent test failure, see: + # https://github.com/simonw/datasette/issues/1802 + if "There is no current event loop in thread" in str(rex): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: diff --git a/tests/test_cli.py b/tests/test_cli.py index d0f6e26c..f0d28037 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,13 +22,6 @@ from unittest import mock import urllib -@pytest.fixture -def ensure_eventloop(): - # Workaround for "Event loop is closed" error - if asyncio.get_event_loop().is_closed(): - asyncio.set_event_loop(asyncio.new_event_loop()) - - def test_inspect_cli(app_client): runner = CliRunner() result = runner.invoke(cli, ["inspect", "fixtures.db"]) @@ -72,7 +65,7 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): ), ) def test_spatialite_error_if_attempt_to_open_spatialite( - ensure_eventloop, spatialite_paths, should_suggest_load_extension + spatialite_paths, should_suggest_load_extension ): with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths): runner = CliRunner() @@ -199,14 +192,14 @@ def test_version(): @pytest.mark.parametrize("invalid_port", ["-1", "0.5", "dog", "65536"]) -def test_serve_invalid_ports(ensure_eventloop, invalid_port): +def test_serve_invalid_ports(invalid_port): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--port", invalid_port]) assert result.exit_code == 2 assert "Invalid value for '-p'" in result.stderr -def test_setting(ensure_eventloop): +def test_setting(): runner = CliRunner() result = runner.invoke( cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] @@ -215,14 +208,14 @@ def test_setting(ensure_eventloop): assert json.loads(result.output)["default_page_size"] == 5 -def test_setting_type_validation(ensure_eventloop): +def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 assert '"default_page_size" should be an integer' in result.stderr -def test_config_deprecated(ensure_eventloop): +def test_config_deprecated(): # The --config option should show a deprecation message runner = CliRunner(mix_stderr=False) result = runner.invoke( @@ -233,14 +226,14 @@ def test_config_deprecated(ensure_eventloop): assert "will be deprecated in" in result.stderr -def test_sql_errors_logged_to_stderr(ensure_eventloop): +def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) assert result.exit_code == 1 assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr -def test_serve_create(ensure_eventloop, tmpdir): +def test_serve_create(tmpdir): runner = CliRunner() db_path = tmpdir / "does_not_exist_yet.db" assert not db_path.exists() @@ -258,7 +251,7 @@ def test_serve_create(ensure_eventloop, tmpdir): assert db_path.exists() -def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): +def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() db_1_path = str(tmpdir / "db.db") @@ -273,7 +266,7 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert {db["name"] for db in databases} == {"db", "db_2"} -def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): +def test_serve_deduplicate_same_database_path(tmpdir): "'datasette db.db db.db' should only attach one database, /db" runner = CliRunner() db_path = str(tmpdir / "db.db") @@ -287,7 +280,7 @@ def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): @pytest.mark.parametrize( "filename", ["test-database (1).sqlite", "database (1).sqlite"] ) -def test_weird_database_names(ensure_eventloop, tmpdir, filename): +def test_weird_database_names(tmpdir, filename): # https://github.com/simonw/datasette/issues/1181 runner = CliRunner() db_path = str(tmpdir / filename) From c9d1943aede436fa3413fd49bc56335cbda4ad07 Mon Sep 17 00:00:00 2001 From: Daniel Rech <dr@netsyno.com> Date: Tue, 6 Sep 2022 02:45:41 +0200 Subject: [PATCH 0769/1866] Fix word break in facets by adding ul.tight-bullets li word-break: break-all (#1794) Thanks, @dmr --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index af3e14d5..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -260,6 +260,7 @@ ul.bullets li { ul.tight-bullets li { list-style-type: disc; margin-bottom: 0; + word-break: break-all; } a.not-underlined { text-decoration: none; From d80775a48d20917633792fdc9525f075d3bc2c7a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 5 Sep 2022 17:44:44 -0700 Subject: [PATCH 0770/1866] Raise error if it's not about loops, refs #1802 --- datasette/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index c6bbdaf0..aeb81687 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -240,6 +240,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + else: + raise self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: From 8430c3bc7dd22b173c1a8c6cd7180e3b31240cd1 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 08:59:19 -0700 Subject: [PATCH 0771/1866] table facet_size in metadata, refs #1804 --- datasette/facets.py | 14 +++++++++++--- tests/test_facets.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b15a758c..e70d42df 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -102,11 +102,19 @@ class Facet: def get_facet_size(self): facet_size = self.ds.setting("default_facet_size") max_returned_rows = self.ds.setting("max_returned_rows") + table_facet_size = None + if self.table: + tables_metadata = self.ds.metadata("tables", database=self.database) or {} + table_metadata = tables_metadata.get(self.table) or {} + if table_metadata: + table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size == "max": - facet_size = max_returned_rows - elif custom_facet_size and custom_facet_size.isdigit(): + if custom_facet_size and custom_facet_size.isdigit(): facet_size = int(custom_facet_size) + elif table_facet_size: + facet_size = table_facet_size + if facet_size == "max": + facet_size = max_returned_rows return min(facet_size, max_returned_rows) async def suggest(self): diff --git a/tests/test_facets.py b/tests/test_facets.py index c28dc43c..cbee23b0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -581,6 +581,23 @@ async def test_facet_size(): ) data5 = response5.json() assert len(data5["facet_results"]["city"]["results"]) == 20 + # Now try messing with facet_size in the table metadata + ds._metadata_local = { + "databases": { + "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}} + } + } + response6 = await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + data6 = response6.json() + assert len(data6["facet_results"]["city"]["results"]) == 6 + # Setting it to max bumps it up to 50 again + ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][ + "facet_size" + ] = "max" + data7 = ( + await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + ).json() + assert len(data7["facet_results"]["city"]["results"]) == 20 def test_other_types_of_facet_in_metadata(): From 303c6c733d95a6133558ec1b468f5bea5827d0d2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 11:05:00 -0700 Subject: [PATCH 0772/1866] Fix for incorrectly handled _facet_size=max, refs #1804 --- datasette/facets.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index e70d42df..7fb0c68b 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -109,12 +109,19 @@ class Facet: if table_metadata: table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size and custom_facet_size.isdigit(): - facet_size = int(custom_facet_size) - elif table_facet_size: - facet_size = table_facet_size - if facet_size == "max": - facet_size = max_returned_rows + if custom_facet_size: + if custom_facet_size == "max": + facet_size = max_returned_rows + elif custom_facet_size.isdigit(): + facet_size = int(custom_facet_size) + else: + # Invalid value, ignore it + custom_facet_size = None + if table_facet_size and not custom_facet_size: + if table_facet_size == "max": + facet_size = max_returned_rows + else: + facet_size = table_facet_size return min(facet_size, max_returned_rows) async def suggest(self): From 0a7815d2038255a0834c955066a2a16c01f707b2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 11:06:49 -0700 Subject: [PATCH 0773/1866] Documentation for facet_size in metadata, closes #1804 --- docs/facets.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/facets.rst b/docs/facets.rst index 2a2eb039..6c9d99bd 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -129,6 +129,22 @@ You can specify :ref:`array <facet_by_json_array>` or :ref:`date <facet_by_date> ] } +You can change the default facet size (the number of results shown for each facet) for a table using ``facet_size``: + +.. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "facets": ["qLegalStatus"], + "facet_size": 10 + } + } + } + } + } Suggested facets ---------------- From d0476897e10249bb4867473722270d02491c2c1f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 11:24:30 -0700 Subject: [PATCH 0774/1866] Fixed Sphinx warning about language = None --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4ef6b768..8965974a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,7 +71,7 @@ release = "" # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From ff9c87197dde8b09f9787ee878804cb6842ea5dc Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 11:26:21 -0700 Subject: [PATCH 0775/1866] Fixed Sphinx warnings on cli-reference page --- docs/cli-reference.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f8419d58..4a8465cb 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -14,7 +14,7 @@ Running ``datasette`` without specifying a command runs the default command, ``d import textwrap def help(args): title = "datasette " + " ".join(args) - cog.out("::\n\n") + cog.out("\n::\n\n") result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) @@ -32,6 +32,7 @@ Running ``datasette --help`` shows a list of all of the available commands. .. [[[cog help(["--help"]) .. ]]] + :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -77,6 +78,7 @@ Once started you can access it at ``http://localhost:8001`` .. [[[cog help(["serve", "--help"]) .. ]]] + :: Usage: datasette serve [OPTIONS] [FILES]... @@ -202,6 +204,7 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam .. [[[cog help(["--help-settings"]) .. ]]] + :: Settings: @@ -258,6 +261,7 @@ Output JSON showing all currently installed plugins, their versions, whether the .. [[[cog help(["plugins", "--help"]) .. ]]] + :: Usage: datasette plugins [OPTIONS] @@ -326,6 +330,7 @@ Would install the `datasette-cluster-map <https://datasette.io/plugins/datasette .. [[[cog help(["install", "--help"]) .. ]]] + :: Usage: datasette install [OPTIONS] PACKAGES... @@ -349,6 +354,7 @@ Uninstall one or more plugins. .. [[[cog help(["uninstall", "--help"]) .. ]]] + :: Usage: datasette uninstall [OPTIONS] PACKAGES... @@ -374,6 +380,7 @@ Additional deployment targets can be added by plugins that use the :ref:`plugin_ .. [[[cog help(["publish", "--help"]) .. ]]] + :: Usage: datasette publish [OPTIONS] COMMAND [ARGS]... @@ -402,6 +409,7 @@ See :ref:`publish_cloud_run`. .. [[[cog help(["publish", "cloudrun", "--help"]) .. ]]] + :: Usage: datasette publish cloudrun [OPTIONS] [FILES]... @@ -459,6 +467,7 @@ See :ref:`publish_heroku`. .. [[[cog help(["publish", "heroku", "--help"]) .. ]]] + :: Usage: datasette publish heroku [OPTIONS] [FILES]... @@ -507,6 +516,7 @@ Package SQLite files into a Datasette Docker container, see :ref:`cli_package`. .. [[[cog help(["package", "--help"]) .. ]]] + :: Usage: datasette package [OPTIONS] FILES... @@ -560,6 +570,7 @@ This performance optimization is used automatically by some of the ``datasette p .. [[[cog help(["inspect", "--help"]) .. ]]] + :: Usage: datasette inspect [OPTIONS] [FILES]... From d0737e4de51ce178e556fc011ccb8cc46bbb6359 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 16:50:43 -0700 Subject: [PATCH 0776/1866] truncate_cells_html now affects URLs too, refs #1805 --- datasette/utils/__init__.py | 10 ++++++++++ datasette/views/database.py | 11 ++++++++--- datasette/views/table.py | 8 ++++++-- tests/fixtures.py | 9 +++++---- tests/test_api.py | 2 +- tests/test_table_api.py | 11 +++++++---- tests/test_table_html.py | 11 +++++++++++ tests/test_utils.py | 20 ++++++++++++++++++++ 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bbaa0510..2bdea673 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1167,3 +1167,13 @@ def resolve_routes(routes, path): if match is not None: return match, view return None, None + + +def truncate_url(url, length): + if (not length) or (len(url) <= length): + return url + bits = url.rsplit(".", 1) + if len(bits) == 2 and 1 <= len(bits[1]) <= 4 and "/" not in bits[1]: + rest, ext = bits + return rest[: length - 1 - len(ext)] + "…." + ext + return url[: length - 1] + "…" diff --git a/datasette/views/database.py b/datasette/views/database.py index 77632b9d..fc344245 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -20,6 +20,7 @@ from datasette.utils import ( path_with_format, path_with_removed_args, sqlite3, + truncate_url, InvalidSql, ) from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden @@ -371,6 +372,7 @@ class QueryView(DataView): async def extra_template(): display_rows = [] + truncate_cells = self.ds.setting("truncate_cells_html") for row in results.rows if results else []: display_row = [] for column, value in zip(results.columns, row): @@ -396,9 +398,12 @@ class QueryView(DataView): if value in ("", None): display_value = Markup(" ") elif is_url(str(display_value).strip()): - display_value = Markup( - '<a href="{url}">{url}</a>'.format( - url=escape(value.strip()) + display_value = markupsafe.Markup( + '<a href="{url}">{truncated_url}</a>'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif isinstance(display_value, bytes): diff --git a/datasette/views/table.py b/datasette/views/table.py index 49c30c9c..60c092f9 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -24,6 +24,7 @@ from datasette.utils import ( path_with_removed_args, path_with_replaced_args, to_css_class, + truncate_url, urlsafe_components, value_as_boolean, ) @@ -966,8 +967,11 @@ async def display_columns_and_rows( display_value = markupsafe.Markup(" ") elif is_url(str(value).strip()): display_value = markupsafe.Markup( - '<a href="{url}">{url}</a>'.format( - url=markupsafe.escape(value.strip()) + '<a href="{url}">{truncated_url}</a>'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif column in table_metadata.get("units", {}) and value != "": diff --git a/tests/fixtures.py b/tests/fixtures.py index c145ac78..82d8452e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -598,23 +598,24 @@ CREATE TABLE roadside_attractions ( pk integer primary key, name text, address text, + url text, latitude real, longitude real ); INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", 37.0167, -122.0024 ); INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", 37.3184, -121.9511 ); INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, 37.5793, -122.3442 ); INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725 ); diff --git a/tests/test_api.py b/tests/test_api.py index f6db2f9d..7a2bf91f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -339,7 +339,7 @@ def test_database_page(app_client): }, { "name": "roadside_attractions", - "columns": ["pk", "name", "address", "latitude", "longitude"], + "columns": ["pk", "name", "address", "url", "latitude", "longitude"], "primary_keys": ["pk"], "count": 4, "hidden": False, diff --git a/tests/test_table_api.py b/tests/test_table_api.py index e56a72b5..0db04434 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -615,11 +615,12 @@ def test_table_through(app_client): response = app_client.get( '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' ) - assert [ + assert response.json["rows"] == [ [ 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + None, 37.5793, -122.3442, ], @@ -627,13 +628,15 @@ def test_table_through(app_client): 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725, ], - ] == response.json["rows"] + ] + assert ( - 'where roadside_attraction_characteristics.characteristic_id = "1"' - == response.json["human_description_en"] + response.json["human_description_en"] + == 'where roadside_attraction_characteristics.characteristic_id = "1"' ) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index f3808ea3..8e37468f 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -69,6 +69,17 @@ def test_table_cell_truncation(): td.string for td in table.findAll("td", {"class": "col-neighborhood-b352a7"}) ] + # URLs should be truncated too + response2 = client.get("/fixtures/roadside_attractions") + assert response2.status == 200 + table = Soup(response2.body, "html.parser").find("table") + tds = table.findAll("td", {"class": "col-url"}) + assert [str(td) for td in tds] == [ + '<td class="col-url type-str"><a href="https://www.mysteryspot.com/">http…</a></td>', + '<td class="col-url type-str"><a href="https://winchestermysteryhouse.com/">http…</a></td>', + '<td class="col-url type-none">\xa0</td>', + '<td class="col-url type-str"><a href="https://www.bigfootdiscoveryproject.com/">http…</a></td>', + ] def test_add_filter_redirects(app_client): diff --git a/tests/test_utils.py b/tests/test_utils.py index df788767..d71a612d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -626,3 +626,23 @@ def test_tilde_encoding(original, expected): assert actual == expected # And test round-trip assert original == utils.tilde_decode(actual) + + +@pytest.mark.parametrize( + "url,length,expected", + ( + ("https://example.com/", 5, "http…"), + ("https://example.com/foo/bar", 15, "https://exampl…"), + ("https://example.com/foo/bar/baz.jpg", 30, "https://example.com/foo/ba….jpg"), + # Extensions longer than 4 characters are not treated specially: + ("https://example.com/foo/bar/baz.jpeg2", 30, "https://example.com/foo/bar/b…"), + ( + "https://example.com/foo/bar/baz.jpeg2", + None, + "https://example.com/foo/bar/baz.jpeg2", + ), + ), +) +def test_truncate_url(url, length, expected): + actual = utils.truncate_url(url, length) + assert actual == expected From 5aa359b86907d11b3ee601510775a85a90224da8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 16:58:30 -0700 Subject: [PATCH 0777/1866] Apply cell truncation on query page too, refs #1805 --- datasette/views/database.py | 7 ++++++- tests/test_html.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fc344245..affbc540 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,12 @@ class QueryView(DataView): "" if len(value) == 1 else "s", ) ) - + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = ( + display_value[:truncate_cells] + "\u2026" + ) display_row.append(display_value) display_rows.append(display_row) diff --git a/tests/test_html.py b/tests/test_html.py index d6e969ad..bf915247 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -186,6 +186,25 @@ def test_row_page_does_not_truncate(): ] +def test_query_page_truncates(): + with make_app_client(settings={"truncate_cells_html": 5}) as client: + response = client.get( + "/fixtures?" + + urllib.parse.urlencode( + { + "sql": "select 'this is longer than 5' as a, 'https://example.com/' as b" + } + ) + ) + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + tds = table.findAll("td") + assert [str(td) for td in tds] == [ + '<td class="col-a">this …</td>', + '<td class="col-b"><a href="https://example.com/">http…</a></td>', + ] + + @pytest.mark.parametrize( "path,expected_classes", [ From bf8d84af5422606597be893cedd375020cb2b369 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 6 Sep 2022 20:34:59 -0700 Subject: [PATCH 0778/1866] word-wrap: anywhere on links in cells, refs #1805 --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 712b9925..08b724f6 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,6 +446,7 @@ th { } table a:link { text-decoration: none; + word-wrap: anywhere; } .rows-and-columns td:before { display: block; From fb7e70d5e72a951efe4b29ad999d8915c032d021 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 9 Sep 2022 09:19:20 -0700 Subject: [PATCH 0779/1866] Database(is_mutable=) now defaults to True, closes #1808 Refs https://github.com/simonw/datasette-upload-dbs/issues/6 --- datasette/database.py | 3 +-- docs/internals.rst | 9 +++++---- tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index fa558045..44467370 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -28,7 +28,7 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None ): self.name = None self.route = None @@ -39,7 +39,6 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.is_mutable = True self.hash = None self.cached_size = None self._cached_table_counts = None diff --git a/docs/internals.rst b/docs/internals.rst index 20797e98..adeec1d8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -426,12 +426,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` Database( datasette, path="path/to/my-new-database.db", - is_mutable=True, ) ) This will add a mutable database and serve it at ``/my-new-database``. +Use ``is_mutable=False`` to add an immutable database. + ``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: .. code-block:: python @@ -671,8 +672,8 @@ Instances of the ``Database`` class can be used to execute queries against attac .. _database_constructor: -Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) ----------------------------------------------------------------------------- +Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None) +--------------------------------------------------------------------------- The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. @@ -685,7 +686,7 @@ The arguments are as follows: Path to a SQLite database file on disk. ``is_mutable`` - boolean - Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + Set this to ``False`` to cause Datasette to open the file in immutable mode. ``is_memory`` - boolean Use this to create non-shared memory connections. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 551f67e1..9e81c1d6 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -499,6 +499,7 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): + assert Database(app_client.ds, is_memory=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 1dc14cab..249920fe 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -58,7 +58,7 @@ async def test_datasette_constructor(): "route": "_memory", "path": None, "size": 0, - "is_mutable": False, + "is_mutable": True, "is_memory": True, "hash": None, } From 610425460b519e9c16d386cb81aa081c9d730ef0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 10 Sep 2022 14:24:26 -0700 Subject: [PATCH 0780/1866] Add --nolock to the README Chrome demo Refs #1744 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1af20129..af95b85e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From b40872f5e5ae5dad331c58f75451e2d206565196 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 14 Sep 2022 14:31:54 -0700 Subject: [PATCH 0781/1866] prepare_jinja2_environment(datasette) argument, refs #1809 --- datasette/app.py | 2 +- datasette/hookspecs.py | 2 +- docs/plugin_hooks.rst | 9 +++++++-- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 5 +++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index aeb81687..db686670 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -345,7 +345,7 @@ class Datasette: self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env) + pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) self._register_renderers() self._permission_checks = collections.deque(maxlen=200) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a5fb536f..34e19664 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -26,7 +26,7 @@ def prepare_connection(conn, database, datasette): @hookspec -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): """Modify Jinja2 template environment e.g. register custom template tags""" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 30bd75b7..62ec5c90 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -61,12 +61,15 @@ Examples: `datasette-jellyfish <https://datasette.io/plugins/datasette-jellyfish .. _plugin_hook_prepare_jinja2_environment: -prepare_jinja2_environment(env) -------------------------------- +prepare_jinja2_environment(env, datasette) +------------------------------------------ ``env`` - jinja2 Environment The template environment that is being prepared +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` + This hook is called with the Jinja2 environment that is used to evaluate Datasette HTML templates. You can use it to do things like `register custom template filters <http://jinja.pocoo.org/docs/2.10/api/#custom-filters>`_, for @@ -85,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +Examples: `datasette-edit-templates <https://datasette.io/plugins/datasette-edit-templates>`_ + .. _plugin_hook_extra_template_vars: extra_template_vars(template, database, table, columns, view_name, request, datasette) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 53613b7d..d49a7a34 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -142,8 +142,9 @@ def extra_template_vars( @hookimpl -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO @hookimpl diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 948a40b8..590d88f6 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -545,11 +545,12 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): + app_client.ds._HELLO = "HI" template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341" == rendered + assert "Hello there, 3,412,341, HI" == rendered def test_hook_publish_subcommand(): From 2ebcffe2226ece2a5a86722790d486a480338632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 12:50:52 -0700 Subject: [PATCH 0782/1866] Bump furo from 2022.6.21 to 2022.9.15 (#1812) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.6.21 to 2022.9.15. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.06.21...2022.09.15) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92fa60d0..afcba1f0 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.6.21", + "furo==2022.9.15", "sphinx-autobuild", "codespell", "blacken-docs", From ddc999ad1296e8c69cffede3e367dda059b8adad Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 16 Sep 2022 20:38:15 -0700 Subject: [PATCH 0783/1866] Async support for prepare_jinja2_environment, closes #1809 --- datasette/app.py | 22 ++++++++++++++--- datasette/utils/testing.py | 1 + docs/plugin_hooks.rst | 2 ++ docs/testing_plugins.rst | 30 ++++++++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 10 ++++++-- tests/plugins/my_plugin_2.py | 6 +++++ tests/test_internals_datasette_client.py | 6 +++-- tests/test_plugins.py | 6 +++-- tests/test_routes.py | 1 + 10 files changed, 76 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index db686670..ea3e7b43 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -208,6 +208,7 @@ class Datasette: crossdb=False, nolock=False, ): + self._startup_invoked = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -344,9 +345,6 @@ class Datasette: self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class - # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) - self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) @@ -389,8 +387,16 @@ class Datasette: return Urls(self) async def invoke_startup(self): + # This must be called for Datasette to be in a usable state + if self._startup_invoked: + return + for hook in pm.hook.prepare_jinja2_environment( + env=self.jinja_env, datasette=self + ): + await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -933,6 +939,8 @@ class Datasette: async def render_template( self, templates, context=None, request=None, view_name=None ): + if not self._startup_invoked: + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -1495,34 +1503,42 @@ class DatasetteClient: return path async def get(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): + await self.ds.invoke_startup() avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient(app=self.app) as client: return await client.request( diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 640c94e6..b28fc575 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -147,6 +147,7 @@ class TestClient: content_type=None, if_none_match=None, ): + await self.ds.invoke_startup() headers = headers or {} if content_type: headers["content-type"] = content_type diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 62ec5c90..f208e727 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -88,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +This function can return an awaitable function if it needs to run any async code. + Examples: `datasette-edit-templates <https://datasette.io/plugins/datasette-edit-templates>`_ .. _plugin_hook_extra_template_vars: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 992b4b0e..41f50e56 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -52,6 +52,36 @@ Then run the tests using pytest like so:: pytest +.. _testing_plugins_datasette_test_instance: + +Setting up a Datasette test instance +------------------------------------ + +The above example shows the easiest way to start writing tests against a Datasette instance: + +.. code-block:: python + + from datasette.app import Datasette + import pytest + + + @pytest.mark.asyncio + async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + +Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this: + +.. code-block:: python + + datasette = Datasette(memory=True) + await datasette.invoke_startup() + +This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. + +If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you. + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/fixtures.py b/tests/fixtures.py index 82d8452e..5a875cd2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -71,6 +71,7 @@ EXPECTED_PLUGINS = [ "handle_exception", "menu_links", "permission_allowed", + "prepare_jinja2_environment", "register_routes", "render_cell", "startup", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index d49a7a34..1a41de38 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -143,8 +143,14 @@ def extra_template_vars( @hookimpl def prepare_jinja2_environment(env, datasette): - env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" - env.filters["to_hello"] = lambda s: datasette._HELLO + async def select_times_three(s): + db = datasette.get_database() + return (await db.execute("select 3 * ?", [int(s)])).first()[0] + + async def inner(): + env.filters["select_times_three"] = select_times_three + + return inner @hookimpl diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 4df02343..cee80703 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -126,6 +126,12 @@ def permission_allowed(datasette, actor, action): return inner +@hookimpl +def prepare_jinja2_environment(env, datasette): + env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO + + @hookimpl def startup(datasette): async def inner(): diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 8c5b5bd3..497bf475 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,10 +1,12 @@ from .fixtures import app_client import httpx import pytest +import pytest_asyncio -@pytest.fixture -def datasette(app_client): +@pytest_asyncio.fixture +async def datasette(app_client): + await app_client.ds.invoke_startup() return app_client.ds diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 590d88f6..0ae3abf3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -546,11 +546,13 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): app_client.ds._HELLO = "HI" + await app_client.ds.invoke_startup() template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", + {"a": 3412341, "b": 5}, ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341, HI" == rendered + assert "Hello there, 3,412,341, HI, 15" == rendered def test_hook_publish_subcommand(): diff --git a/tests/test_routes.py b/tests/test_routes.py index 5ae55d21..d467abe1 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -59,6 +59,7 @@ def test_routes(routes, path, expected_class, expected_matches): @pytest_asyncio.fixture async def ds_with_route(): ds = Datasette() + await ds.invoke_startup() ds.remove_database("_memory") db = Database(ds, is_memory=True, memory_name="route-name-db") ds.add_database(db, name="original-name", route="custom-route-name") From df851c117db031dec50dd4ef1ca34745920ac77a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 19 Sep 2022 16:46:39 -0700 Subject: [PATCH 0784/1866] Validate settings.json keys on startup, closes #1816 Refs #1814 --- datasette/app.py | 4 ++++ tests/test_config_dir.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ea3e7b43..8873ce28 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -292,6 +292,10 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not settings: settings = json.loads((config_dir / "settings.json").read_text()) + # Validate those settings + for key in settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{key}' in settings.json") self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index fe927c42..e365515b 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -5,6 +5,7 @@ import pytest from datasette.app import Datasette from datasette.cli import cli from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient from click.testing import CliRunner @@ -27,9 +28,8 @@ body { margin-top: 3em} @pytest.fixture(scope="session") -def config_dir_client(tmp_path_factory): +def config_dir(tmp_path_factory): config_dir = tmp_path_factory.mktemp("config-dir") - plugins_dir = config_dir / "plugins" plugins_dir.mkdir() (plugins_dir / "hooray.py").write_text(PLUGIN, "utf-8") @@ -77,7 +77,23 @@ def config_dir_client(tmp_path_factory): ), "utf-8", ) + return config_dir + +def test_invalid_settings(config_dir): + previous = (config_dir / "settings.json").read_text("utf-8") + (config_dir / "settings.json").write_text( + json.dumps({"invalid": "invalid-setting"}), "utf-8" + ) + try: + with pytest.raises(StartupError): + ds = Datasette([], config_dir=config_dir) + finally: + (config_dir / "settings.json").write_text(previous, "utf-8") + + +@pytest.fixture(scope="session") +def config_dir_client(config_dir): ds = Datasette([], config_dir=config_dir) yield _TestClient(ds) From cb1e093fd361b758120aefc1a444df02462389a3 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 19 Sep 2022 18:15:40 -0700 Subject: [PATCH 0785/1866] Fixed error message, closes #1816 --- datasette/app.py | 4 +++- tests/test_config_dir.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8873ce28..03d1dacc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -295,7 +295,9 @@ class Datasette: # Validate those settings for key in settings: if key not in DEFAULT_SETTINGS: - raise StartupError("Invalid setting '{key}' in settings.json") + raise StartupError( + "Invalid setting '{}' in settings.json".format(key) + ) self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index e365515b..f5ecf0d6 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -86,8 +86,9 @@ def test_invalid_settings(config_dir): json.dumps({"invalid": "invalid-setting"}), "utf-8" ) try: - with pytest.raises(StartupError): + with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) + assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" finally: (config_dir / "settings.json").write_text(previous, "utf-8") From 212137a90b4291db9605e039f198564dae59c5d0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 26 Sep 2022 14:14:25 -0700 Subject: [PATCH 0786/1866] Release 0.63a0 Refs #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816 --- datasette/version.py | 2 +- docs/changelog.rst | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0453346c..e5ad585f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62" +__version__ = "0.63a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index f9dcc980..bd93f4cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,23 @@ Changelog ========= +.. _v0_63a0: + +0.63a0 (2022-09-26) +------------------- + +- The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 <https://github.com/simonw/datasette/pull/1789>`__) +- New tutorial: `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__. +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) +- ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) +- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) +- More detailed command descriptions on the :ref:`CLI reference <cli_reference>` page. (:issue:`1787`) +- Datasette no longer enforces upper bounds on its depenedencies. (:issue:`1800`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 <https://github.com/simonw/datasette/pull/1794>`__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + .. _v0_62: 0.62 (2022-08-14) From 5f9f567acbc58c9fcd88af440e68034510fb5d2b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Mon, 26 Sep 2022 16:06:01 -0700 Subject: [PATCH 0787/1866] Show SQL query when reporting time limit error, closes #1819 --- datasette/database.py | 5 ++++- datasette/views/base.py | 21 +++++++++++++-------- tests/test_api.py | 12 +++++++++++- tests/test_html.py | 10 +++++++--- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 44467370..46094bd7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -476,7 +476,10 @@ class WriteTask: class QueryInterrupted(Exception): - pass + def __init__(self, e, sql, params): + self.e = e + self.sql = sql + self.params = params class MultipleValues(Exception): diff --git a/datasette/views/base.py b/datasette/views/base.py index 221e1882..67aa3a42 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,10 +1,12 @@ import asyncio import csv import hashlib -import re import sys +import textwrap import time import urllib +from markupsafe import escape + import pint @@ -24,11 +26,9 @@ from datasette.utils import ( path_with_removed_args, path_with_format, sqlite3, - HASH_LENGTH, ) from datasette.utils.asgi import ( AsgiStream, - Forbidden, NotFound, Response, BadRequest, @@ -371,13 +371,18 @@ class DataView(BaseView): ) = response_or_template_contexts else: data, extra_template_data, templates = response_or_template_contexts - except QueryInterrupted: + except QueryInterrupted as ex: raise DatasetteError( - """ - SQL query took too long. The time limit is controlled by the + textwrap.dedent( + """ + <p>SQL query took too long. The time limit is controlled by the <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a> - configuration option. - """, + configuration option.</p> + <pre>{}</pre> + """.format( + escape(ex.sql) + ) + ).strip(), title="SQL Interrupted", status=400, message_is_html=True, diff --git a/tests/test_api.py b/tests/test_api.py index 7a2bf91f..ad74d16e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -656,7 +656,17 @@ def test_custom_sql(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") assert 400 == response.status - assert "SQL Interrupted" == response.json["title"] + assert response.json == { + "ok": False, + "error": ( + "<p>SQL query took too long. The time limit is controlled by the\n" + '<a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>\n' + "configuration option.</p>\n" + "<pre>select sleep(0.5)</pre>" + ), + "status": 400, + "title": "SQL Interrupted", + } def test_custom_sql_time_limit(app_client): diff --git a/tests/test_html.py b/tests/test_html.py index bf915247..a99b0b6c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -168,10 +168,14 @@ def test_disallowed_custom_sql_pragma(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)") assert 400 == response.status - expected_html_fragment = """ + expected_html_fragments = [ + """ <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a> - """.strip() - assert expected_html_fragment in response.text + """.strip(), + "<pre>select sleep(0.5)</pre>", + ] + for expected_html_fragment in expected_html_fragments: + assert expected_html_fragment in response.text def test_row_page_does_not_truncate(): From 7fb4ea4e39a15e1f7d3202949794d98af1cfa272 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 27 Sep 2022 21:06:40 -0700 Subject: [PATCH 0788/1866] Update note about render_cell signature, refs #1826 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f208e727..c9cab8ab 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -9,7 +9,7 @@ Each plugin can implement one or more hooks using the ``@hookimpl`` decorator ag When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook. -For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(value, column, table, database, datasette)``: +For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(row, value, column, table, database, datasette)``: .. code-block:: python From 984b1df12cf19a6731889fc0665bb5f622e07b7c Mon Sep 17 00:00:00 2001 From: Adam Simpson <adam@adamsimpson.net> Date: Wed, 28 Sep 2022 00:21:36 -0400 Subject: [PATCH 0789/1866] Add documentation for serving via OpenRC (#1825) * Add documentation for serving via OpenRC --- docs/deploying.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index d4ad8836..c8552758 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -74,18 +74,30 @@ Once the service has started you can confirm that Datasette is running on port 8 curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version -Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. +Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx`` - see :ref:`deploying_proxy`. -Ubuntu offer `a tutorial on installing nginx <https://ubuntu.com/tutorials/install-and-configure-nginx#1-overview>`__. Once it is installed you can add configuration to proxy traffic through to Datasette that looks like this:: +.. _deploying_openrc: - server { - server_name mysubdomain.myhost.net; +Running Datasette using OpenRC +=============================== +OpenRC is the service manager on non-systemd Linux distributions like `Alpine Linux <https://www.alpinelinux.org/>`__ and `Gentoo <https://www.gentoo.org/>`__. - location / { - proxy_pass http://127.0.0.1:8000/; - proxy_set_header Host $host; - } - } +Create an init script at ``/etc/init.d/datasette`` with the following contents: + +.. code-block:: sh + + #!/sbin/openrc-run + + name="datasette" + command="datasette" + command_args="serve -h 0.0.0.0 /path/to/db.db" + command_background=true + pidfile="/run/${RC_SVCNAME}.pid" + +You then need to configure the service to run at boot and start it:: + + rc-update add datasette + rc-service datasette start .. _deploying_buildpacks: From 34defdc10aa293294ca01cfab70780755447e1d7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 28 Sep 2022 17:39:36 -0700 Subject: [PATCH 0790/1866] Browse the plugins directory --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 01ee8c90..a3fc88ec 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -234,7 +234,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D - ``/-/upload-excel`` -Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic <https://github.com/topics/datasette-plugin>`__ on GitHub. +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory <https://datasette.io/plugins>`. If your plugin includes functionality that relates to a specific database you could also register a URL route like this: From c92c4318e9892101f75fa158410c0a12c1d80b6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:55:40 -0700 Subject: [PATCH 0791/1866] Bump furo from 2022.9.15 to 2022.9.29 (#1827) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.9.15 to 2022.9.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.09.15...2022.09.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afcba1f0..fe258adb 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.9.15", + "furo==2022.9.29", "sphinx-autobuild", "codespell", "blacken-docs", From 883e326dd6ef95f854f7750ef2d4b0e17082fa96 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 2 Oct 2022 14:26:16 -0700 Subject: [PATCH 0792/1866] Drop word-wrap: anywhere, refs #1828, #1805 --- datasette/static/app.css | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 08b724f6..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,7 +446,6 @@ th { } table a:link { text-decoration: none; - word-wrap: anywhere; } .rows-and-columns td:before { display: block; From 4218c9cd742b79b1e3cb80878e42b7e39d16ded2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 4 Oct 2022 11:45:36 -0700 Subject: [PATCH 0793/1866] reST markup fix --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c9cab8ab..832a76b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -268,7 +268,7 @@ you have one: def extra_js_urls(): return ["/-/static-plugins/your-plugin/app.js"] -Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. +Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. If your code uses `JavaScript modules <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules>`__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. From b6ba117b7978b58b40e3c3c2b723b92c3010ed53 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 4 Oct 2022 18:25:52 -0700 Subject: [PATCH 0794/1866] Clarify request or None for two hooks --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 832a76b0..b61f953a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1281,7 +1281,7 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor <authentication_actor>`. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1330,7 +1330,7 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. From bbf33a763537a1d913180b22bd3b5fe4a5e5b252 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 4 Oct 2022 21:32:11 -0700 Subject: [PATCH 0795/1866] Test for bool(results), closes #1832 --- tests/test_internals_database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 9e81c1d6..4e33beed 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -30,6 +30,14 @@ async def test_results_first(db): assert isinstance(row, sqlite3.Row) +@pytest.mark.asyncio +@pytest.mark.parametrize("expected", (True, False)) +async def test_results_bool(db, expected): + where = "" if expected else "where pk = 0" + results = await db.execute("select * from facetable {}".format(where)) + assert bool(results) is expected + + @pytest.mark.parametrize( "query,expected", [ From eff112498ecc499323c26612d707908831446d25 Mon Sep 17 00:00:00 2001 From: Forest Gregg <fgregg@users.noreply.github.com> Date: Thu, 6 Oct 2022 16:06:06 -0400 Subject: [PATCH 0796/1866] Useuse inspect data for hash and file size on startup Thanks, @fgregg Closes #1834 --- datasette/database.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 46094bd7..d75bd70c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -48,9 +48,13 @@ class Database: self._read_connection = None self._write_connection = None if not self.is_mutable and not self.is_memory: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.hash = self.ds.inspect_data[self.name]["hash"] + self.cached_size = self.ds.inspect_data[self.name]["size"] + else: + p = Path(path) + self.hash = inspect_hash(p) + self.cached_size = p.stat().st_size @property def cached_table_counts(self): From b7fec7f9020b79c1fe60cc5a2def86b50eeb5af9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 7 Oct 2022 16:03:09 -0700 Subject: [PATCH 0797/1866] .sqlite/.sqlite3 extensions for config directory mode Closes #1646 --- datasette/app.py | 5 ++++- docs/settings.rst | 2 +- tests/test_config_dir.py | 11 +++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 03d1dacc..32a911c2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -217,7 +217,10 @@ class Datasette: self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) if config_dir: - self.files += tuple([str(p) for p in config_dir.glob("*.db")]) + db_files = [] + for ext in ("db", "sqlite", "sqlite3"): + db_files.extend(config_dir.glob("*.{}".format(ext))) + self.files += tuple(str(f) for f in db_files) if ( config_dir and (config_dir / "inspect-data.json").exists() diff --git a/docs/settings.rst b/docs/settings.rst index 8437fb04..a6d50543 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -46,7 +46,7 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. -* ``*.db`` - SQLite database files that will be served by Datasette +* ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index f5ecf0d6..c2af3836 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -49,7 +49,7 @@ def config_dir(tmp_path_factory): (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") - for dbname in ("demo.db", "immutable.db"): + for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) db.executescript( """ @@ -151,12 +151,11 @@ def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status databases = response.json - assert 2 == len(databases) + assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) - assert "demo" == databases[0]["name"] - assert databases[0]["is_mutable"] - assert "immutable" == databases[1]["name"] - assert not databases[1]["is_mutable"] + for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): + assert expected_name == db["name"] + assert db["is_mutable"] == (expected_name != "immutable") @pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml")) From 1a5e5f2aa951e5bd731067a49819efba68fbe8ef Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 13 Oct 2022 14:42:52 -0700 Subject: [PATCH 0798/1866] Refactor breadcrumbs to respect permissions, refs #1831 --- datasette/app.py | 40 ++++++++++++++++++++++ datasette/templates/_crumbs.html | 15 ++++++++ datasette/templates/base.html | 4 +-- datasette/templates/database.html | 9 ----- datasette/templates/error.html | 7 ---- datasette/templates/logout.html | 7 ---- datasette/templates/permissions_debug.html | 7 ---- datasette/templates/query.html | 8 ++--- datasette/templates/row.html | 9 ++--- datasette/templates/show_json.html | 7 ---- datasette/templates/table.html | 8 ++--- tests/test_permissions.py | 1 + tests/test_plugins.py | 2 +- 13 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 datasette/templates/_crumbs.html diff --git a/datasette/app.py b/datasette/app.py index 32a911c2..5fa4955c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -631,6 +631,44 @@ class Datasette: else: return [] + async def _crumb_items(self, request, table=None, database=None): + crumbs = [] + # Top-level link + if await self.permission_allowed( + actor=request.actor, action="view-instance", default=True + ): + crumbs.append({"href": self.urls.instance(), "label": "home"}) + # Database link + if database: + if await self.permission_allowed( + actor=request.actor, + action="view-database", + resource=database, + default=True, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) + # Table link + if table: + assert database, "table= requires database=" + if await self.permission_allowed( + actor=request.actor, + action="view-table", + resource=(database, table), + default=True, + ): + crumbs.append( + { + "href": self.urls.table(database, table), + "label": table, + } + ) + return crumbs + async def permission_allowed(self, actor, action, resource=None, default=False): """Check permissions using the permissions_allowed plugin hook""" result = None @@ -1009,6 +1047,8 @@ class Datasette: template_context = { **context, **{ + "request": request, + "crumb_items": self._crumb_items, "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html new file mode 100644 index 00000000..bd1ff0da --- /dev/null +++ b/datasette/templates/_crumbs.html @@ -0,0 +1,15 @@ +{% macro nav(request, database=None, table=None) -%} +{% if crumb_items is defined %} + {% set items=crumb_items(request=request, database=database, table=table) %} + {% if items %} + <p class="crumbs"> + {% for item in items %} + <a href="{{ item.href }}">{{ item.label }}</a> + {% if not loop.last %} + / + {% endif %} + {% endfor %} + </p> + {% endif %} +{% endif %} +{%- endmacro %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c3a71acb..87c939ac 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,4 +1,4 @@ -<!DOCTYPE html> +{% import "_crumbs.html" as crumbs with context %}<!DOCTYPE html> <html> <head> <title>{% block title %}{% endblock %} @@ -17,7 +17,7 @@

    -
    + {% if not column.sortable %} {{ column.name }} {% else %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 6b859132..5474bf35 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -5,6 +5,7 @@ {% block extra_head %} {{ super() }} + {% endblock %} @@ -111,7 +110,7 @@

    ✎ View and edit SQL

    {% endif %} -

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}

    +

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}

    {% if suggested_facets %}

    @@ -160,10 +159,10 @@

    Advanced export

    JSON shape: - default, - array, - newline-delimited{% if primary_keys %}, - object + default, + array, + newline-delimited{% if primary_keys %}, + object {% endif %}

    diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 6fc4c633..8a8810e7 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -1,23 +1,39 @@ -from datasette.utils import MultiParams -from asgiref.testing import ApplicationCommunicator from asgiref.sync import async_to_sync -from urllib.parse import unquote, quote, urlencode -from http.cookies import SimpleCookie +from urllib.parse import urlencode import json +# These wrapper classes pre-date the introduction of +# datasette.client and httpx to Datasette. They could +# be removed if the Datasette tests are modified to +# call datasette.client directly. + class TestResponse: - def __init__(self, status, headers, body): - self.status = status - self.headers = headers - self.body = body + def __init__(self, httpx_response): + self.httpx_response = httpx_response + + @property + def status(self): + return self.httpx_response.status_code + + @property + def headers(self): + return self.httpx_response.headers + + @property + def body(self): + return self.httpx_response.content @property def cookies(self): - cookie = SimpleCookie() - for header in self.headers.getlist("set-cookie"): - cookie.load(header) - return {key: value.value for key, value in cookie.items()} + return dict(self.httpx_response.cookies) + + def cookie_was_deleted(self, cookie): + return any( + h + for h in self.httpx_response.headers.get_list("set-cookie") + if h.startswith('{}="";'.format(cookie)) + ) @property def json(self): @@ -31,8 +47,8 @@ class TestResponse: class TestClient: max_redirects = 5 - def __init__(self, asgi_app): - self.asgi_app = asgi_app + def __init__(self, ds): + self.ds = ds def actor_cookie(self, actor): return self.ds.sign({"a": actor}, "actor") @@ -94,61 +110,18 @@ class TestClient: post_body=None, content_type=None, ): - query_string = b"" - if "?" in path: - path, _, query_string = path.partition("?") - query_string = query_string.encode("utf8") - if "%" in path: - raw_path = path.encode("latin-1") - else: - raw_path = quote(path, safe="/:,").encode("latin-1") - asgi_headers = [[b"host", b"localhost"]] - if headers: - for key, value in headers.items(): - asgi_headers.append([key.encode("utf-8"), value.encode("utf-8")]) + headers = headers or {} if content_type: - asgi_headers.append((b"content-type", content_type.encode("utf-8"))) - if cookies: - sc = SimpleCookie() - for key, value in cookies.items(): - sc[key] = value - asgi_headers.append([b"cookie", sc.output(header="").encode("utf-8")]) - scope = { - "type": "http", - "http_version": "1.0", - "method": method, - "path": unquote(path), - "raw_path": raw_path, - "query_string": query_string, - "headers": asgi_headers, - } - instance = ApplicationCommunicator(self.asgi_app, scope) - - if post_body: - body = post_body.encode("utf-8") - await instance.send_input({"type": "http.request", "body": body}) - else: - await instance.send_input({"type": "http.request"}) - - # First message back should be response.start with headers and status - messages = [] - start = await instance.receive_output(2) - messages.append(start) - assert start["type"] == "http.response.start" - response_headers = MultiParams( - [(k.decode("utf8"), v.decode("utf8")) for k, v in start["headers"]] + headers["content-type"] = content_type + httpx_response = await self.ds.client.request( + method, + path, + allow_redirects=allow_redirects, + cookies=cookies, + headers=headers, + content=post_body, ) - status = start["status"] - # Now loop until we run out of response.body - body = b"" - while True: - message = await instance.receive_output(2) - messages.append(message) - assert message["type"] == "http.response.body" - body += message["body"] - if not message.get("more_body"): - break - response = TestResponse(status, response_headers, body) + response = TestResponse(httpx_response) if allow_redirects and response.status in (301, 302): assert ( redirect_count < self.max_redirects diff --git a/datasette/views/base.py b/datasette/views/base.py index 3fe2abd5..6cf0e8d9 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -113,6 +113,15 @@ class BaseView: async def options(self, request, *args, **kwargs): return Response.text("Method not allowed", status=405) + async def put(self, request, *args, **kwargs): + return Response.text("Method not allowed", status=405) + + async def patch(self, request, *args, **kwargs): + return Response.text("Method not allowed", status=405) + + async def delete(self, request, *args, **kwargs): + return Response.text("Method not allowed", status=405) + async def dispatch_request(self, request, *args, **kwargs): handler = getattr(self, request.method.lower(), None) return await handler(request, *args, **kwargs) diff --git a/setup.py b/setup.py index ddcd8106..8443fb41 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "click-default-group~=1.2.2", "Jinja2>=2.10.3,<2.12.0", "hupper~=1.9", + "httpx>=0.15", "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", diff --git a/tests/fixtures.py b/tests/fixtures.py index 2f990490..e2a0ae1e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -144,9 +144,7 @@ def make_app_client( template_dir=template_dir, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) - client = TestClient(ds.app()) - client.ds = ds - yield client + yield TestClient(ds) @pytest.fixture(scope="session") @@ -158,9 +156,7 @@ def app_client(): @pytest.fixture(scope="session") def app_client_no_files(): ds = Datasette([]) - client = TestClient(ds.app()) - client.ds = ds - yield client + yield TestClient(ds) @pytest.fixture(scope="session") diff --git a/tests/test_api.py b/tests/test_api.py index c797a1ad..4aa9811c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -739,6 +739,7 @@ def test_table_shape_object_compound_primary_Key(app_client): assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json +@pytest.mark.xfail def test_table_with_slashes_in_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" @@ -1186,6 +1187,7 @@ def test_row_format_in_querystring(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] +@pytest.mark.xfail def test_row_strange_table_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" diff --git a/tests/test_auth.py b/tests/test_auth.py index a4c5cf45..f244f268 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -87,7 +87,8 @@ def test_logout(app_client): cookies={"ds_actor": app_client.actor_cookie({"id": "test"})}, allow_redirects=False, ) - assert "" == response4.cookies["ds_actor"] + # The ds_actor cookie should have been unset + assert response4.cookie_was_deleted("ds_actor") # Should also have set a message messages = app_client.ds.unsign(response4.cookies["ds_messages"], "messages") assert [["You are now logged out", 2]] == messages diff --git a/tests/test_cli.py b/tests/test_cli.py index 7ae9d6e7..09864602 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,8 +108,7 @@ def test_metadata_yaml(): open_browser=False, return_instance=True, ) - client = _TestClient(ds.app()) - client.ds = ds + client = _TestClient(ds) response = client.get("/-/metadata.json") assert {"title": "Hello from YAML"} == response.json diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 430eba16..15c7a5c4 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -76,9 +76,7 @@ def config_dir_client(tmp_path_factory): ) ds = Datasette([], config_dir=config_dir) - client = _TestClient(ds.app()) - client.ds = ds - yield client + yield _TestClient(ds) def test_metadata(config_dir_client): @@ -137,8 +135,7 @@ def test_metadata_yaml(tmp_path_factory, filename): config_dir = tmp_path_factory.mktemp("yaml-config-dir") (config_dir / filename).write_text("title: Title from metadata", "utf-8") ds = Datasette([], config_dir=config_dir) - client = _TestClient(ds.app()) - client.ds = ds + client = _TestClient(ds) response = client.get("/-/metadata.json") assert 200 == response.status assert {"title": "Title from metadata"} == response.json diff --git a/tests/test_html.py b/tests/test_html.py index 02d49b52..c0e3625e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -142,6 +142,7 @@ def test_row_redirects_with_url_hash(app_client_with_hash): assert response.status == 200 +@pytest.mark.xfail def test_row_strange_table_name_with_url_hash(app_client_with_hash): response = app_client_with_hash.get( "/fixtures/table%2Fwith%2Fslashes.csv/3", allow_redirects=False @@ -535,6 +536,7 @@ def test_facets_persist_through_filter_form(app_client): ] +@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_classes", [ @@ -566,6 +568,7 @@ def test_css_classes_on_body(app_client, path, expected_classes): assert classes == expected_classes +@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_considered", [ diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py new file mode 100644 index 00000000..d73fbb06 --- /dev/null +++ b/tests/test_internals_datasette_client.py @@ -0,0 +1,44 @@ +from .fixtures import app_client +import httpx +import pytest + + +@pytest.fixture +def datasette(app_client): + return app_client.ds + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,path,expected_status", + [ + ("get", "/", 200), + ("options", "/", 405), + ("head", "/", 200), + ("put", "/", 405), + ("patch", "/", 405), + ("delete", "/", 405), + ], +) +async def test_client_methods(datasette, method, path, expected_status): + client_method = getattr(datasette.client, method) + response = await client_method(path) + assert isinstance(response, httpx.Response) + assert response.status_code == expected_status + # Try that again using datasette.client.request + response2 = await datasette.client.request(method, path) + assert response2.status_code == expected_status + + +@pytest.mark.asyncio +async def test_client_post(datasette): + response = await datasette.client.post( + "/-/messages", + data={ + "message": "A message", + }, + allow_redirects=False, + ) + assert isinstance(response, httpx.Response) + assert response.status_code == 302 + assert "ds_messages" in response.cookies diff --git a/tests/test_messages.py b/tests/test_messages.py index d17e015c..830244e1 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -25,4 +25,4 @@ def test_messages_are_displayed_and_cleared(app_client): # Messages should be in that HTML assert "xmessagex" in response.text # Cookie should have been set that clears messages - assert "" == response.cookies["ds_messages"] + assert response.cookie_was_deleted("ds_messages") diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 00bedb03..4b3634ab 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -380,9 +380,7 @@ def view_names_client(tmp_path_factory): conn = sqlite3.connect(db_path) conn.executescript(TABLES) return _TestClient( - Datasette( - [db_path], template_dir=str(templates), plugins_dir=str(plugins) - ).app() + Datasette([db_path], template_dir=str(templates), plugins_dir=str(plugins)) ) @@ -748,7 +746,7 @@ def test_hook_register_magic_parameters(restore_working_directory): response = client.post("/data/runme", {}, csrftoken_from=True) assert 200 == response.status actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json - assert [{"rowid": 1, "line": "1.0"}] == actual + assert [{"rowid": 1, "line": "1.1"}] == actual # Now try the GET request against get_uuid response_get = client.get("/data/get_uuid.json?_shape=array") assert 200 == response_get.status From 6e091b14b651d67e0ff41a353d36bbeb1d8ba235 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 09:22:49 -0700 Subject: [PATCH 0049/1866] Run tests against Python 3.9 --- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/test-coverage.yml | 4 ++-- .github/workflows/test.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 6c4b4334..625d16b9 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 84a8be6c..c1909bbe 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.9' - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 0b964c75..1d1cf332 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -14,9 +14,9 @@ jobs: - name: Check out datasette uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74e56e13..a1774213 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 896cc2c6acfefa65c54a162831e7f09159603988 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 09:26:17 -0700 Subject: [PATCH 0050/1866] Replace MockRequest with Request.fake() Close #1004 --- tests/test_facets.py | 20 ++++++++++---------- tests/utils.py | 8 -------- 2 files changed, 10 insertions(+), 18 deletions(-) delete mode 100644 tests/utils.py diff --git a/tests/test_facets.py b/tests/test_facets.py index e3dc3df3..1e19dc3a 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,7 +1,7 @@ from datasette.facets import ColumnFacet, ArrayFacet, DateFacet +from datasette.utils.asgi import Request from datasette.utils import detect_json1 from .fixtures import app_client # noqa -from .utils import MockRequest import pytest @@ -9,7 +9,7 @@ import pytest async def test_column_facet_suggest(app_client): facet = ColumnFacet( app_client.ds, - MockRequest("http://localhost/"), + Request.fake("/"), database="fixtures", sql="select * from facetable", table="facetable", @@ -34,7 +34,7 @@ async def test_column_facet_suggest(app_client): async def test_column_facet_suggest_skip_if_already_selected(app_client): facet = ColumnFacet( app_client.ds, - MockRequest("http://localhost/?_facet=planet_int&_facet=on_earth"), + Request.fake("/?_facet=planet_int&_facet=on_earth"), database="fixtures", sql="select * from facetable", table="facetable", @@ -72,7 +72,7 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client): async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): facet = ColumnFacet( app_client.ds, - MockRequest("http://localhost/"), + Request.fake("/"), database="fixtures", sql="select * from facetable", table="facetable", @@ -94,7 +94,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client): async def test_column_facet_results(app_client): facet = ColumnFacet( app_client.ds, - MockRequest("http://localhost/?_facet=city_id"), + Request.fake("/?_facet=city_id"), database="fixtures", sql="select * from facetable", table="facetable", @@ -146,7 +146,7 @@ async def test_column_facet_results(app_client): async def test_column_facet_from_metadata_cannot_be_hidden(app_client): facet = ColumnFacet( app_client.ds, - MockRequest("http://localhost/"), + Request.fake("/"), database="fixtures", sql="select * from facetable", table="facetable", @@ -200,7 +200,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(app_client): async def test_array_facet_suggest(app_client): facet = ArrayFacet( app_client.ds, - MockRequest("http://localhost/"), + Request.fake("/"), database="fixtures", sql="select * from facetable", table="facetable", @@ -220,7 +220,7 @@ async def test_array_facet_suggest(app_client): async def test_array_facet_suggest_not_if_all_empty_arrays(app_client): facet = ArrayFacet( app_client.ds, - MockRequest("http://localhost/"), + Request.fake("/"), database="fixtures", sql="select * from facetable where tags = '[]'", table="facetable", @@ -234,7 +234,7 @@ async def test_array_facet_suggest_not_if_all_empty_arrays(app_client): async def test_array_facet_results(app_client): facet = ArrayFacet( app_client.ds, - MockRequest("http://localhost/?_facet_array=tags"), + Request.fake("/?_facet_array=tags"), database="fixtures", sql="select * from facetable", table="facetable", @@ -279,7 +279,7 @@ async def test_array_facet_results(app_client): async def test_date_facet_results(app_client): facet = DateFacet( app_client.ds, - MockRequest("http://localhost/?_facet_date=created"), + Request.fake("/?_facet_date=created"), database="fixtures", sql="select * from facetable", table="facetable", diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index 8947956b..00000000 --- a/tests/utils.py +++ /dev/null @@ -1,8 +0,0 @@ -class MockRequest: - def __init__(self, url): - self.url = url - self.path = "/" + url.split("://")[1].split("/", 1)[1] - self.query_string = "" - if "?" in url: - self.query_string = url.split("?", 1)[1] - self.path = self.path.split("?")[0] From 6421ca2b22a8ebd801ca17b2ea38a98d353f1faa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 09:28:17 -0700 Subject: [PATCH 0051/1866] Use actions/setup-python@v2 to deploy latest This should fix an error with Python 3.9. --- .github/workflows/deploy-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 625d16b9..55aabb76 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -12,7 +12,7 @@ jobs: - name: Check out datasette uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.9 - uses: actions/cache@v2 From c12b7a5def7028845a54a9fdac4052a87a0a8bb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 10:19:50 -0700 Subject: [PATCH 0052/1866] Documentation for datasette.client, closes #1006 Refs #1000 --- docs/internals.rst | 55 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index bffda3f7..94c142c2 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1,14 +1,15 @@ .. _internals: -Internals for plugins -===================== +======================= + Internals for plugins +======================= Many :ref:`plugin_hooks` are passed objects that provide access to internal Datasette functionality. The interface to these objects should not be considered stable with the exception of methods that are documented here. .. _internals_request: Request object -~~~~~~~~~~~~~~ +============== The request object is passed to various plugin hooks. It represents an incoming HTTP request. It has the following properties: @@ -59,7 +60,7 @@ The object also has two awaitable methods: .. _internals_multiparams: The MultiParams class ---------------------- +===================== ``request.args`` is a ``MultiParams`` object - a dictionary-like object which provides access to querystring parameters that may have multiple values. @@ -89,7 +90,7 @@ Consider the querystring ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` an .. _internals_response: Response class -~~~~~~~~~~~~~~ +============== The ``Response`` class can be returned from view functions that have been registered using the :ref:`plugin_register_routes` hook. @@ -167,7 +168,7 @@ You can use this with :ref:`datasette.sign() ` to set signed coo .. _internals_datasette: Datasette class -~~~~~~~~~~~~~~~ +=============== This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``. @@ -327,10 +328,48 @@ Datasette's flash messaging mechanism allows you to add a message that will be d You can try out these messages (including the different visual styling of the three message types) using the ``/-/messages`` debugging tool. +.. _internals_datasette_client: + +.client +------- + +Plugins can make internal HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins. + +The ``datasette.client`` object is a wrapper around the `HTTPX Python library `__, providing an async-friendly API that is similar to the widely used `Requests library `__. + +It offers the following methods: + +``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response + Execute an internal GET request against that path. + +``await datasette.client.post(path, **kwargs)`` - returns HTTPX Respons + Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters. + +``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response + Execute an internal OPTIONS request. + +``await datasette.client.head(path, **kwargs)`` - returns HTTPX Respons + Execute an internal HEAD request. + +``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response + Execute an internal PUT request. + +``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response + Execute an internal PATCH request. + +``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response + Execute an internal DELETE request. + +``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response + Execute an internal request with the given HTTP method against that path. + +For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. + + .. _internals_database: Database class -~~~~~~~~~~~~~~ +============== Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas. @@ -549,7 +588,7 @@ The ``Database`` class also provides properties and methods for introspecting th .. _internals_csrf: CSRF protection -~~~~~~~~~~~~~~~ +=============== Datasette uses `asgi-csrf `__ to guard against CSRF attacks on form POST submissions. Users receive a ``ds_csrftoken`` cookie which is compared against the ``csrftoken`` form field (or ``x-csrftoken`` HTTP header) for every incoming request. From a61f0e4e1588083c9fe4636b8fb7178477c4c4a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 10:51:03 -0700 Subject: [PATCH 0053/1866] Release 0.50 Refs #1001, #514, #891, #943, #969, #970, #978, #980, #996, #997 Closes #1002 --- README.md | 1 + docs/changelog.rst | 30 +++++++++++++++++------------- docs/internals.rst | 5 ++--- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index cc912a13..5c65d17c 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News + * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. diff --git a/docs/changelog.rst b/docs/changelog.rst index aad86e7b..046f5b4d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,24 +4,28 @@ Changelog ========= -.. _v0_50_a1: +.. _v0_50: -0.50a1 (2020-10-06) -------------------- +0.50 (2020-10-09) +----------------- + +The key new feature in this release is the **column actions** menu on the table page (`#891 `__). This can be used to sort a column in ascending or descending order, facet data by that column or filter the table to just rows that have a value for that column. + +Plugin authors can use the new :ref:`internals_datasette_client` object to make internal HTTP requests from their plugins, allowing them to make use of Datasette's JSON API. (`#943 `__) + +New :ref:`deploying` documentation with guides for deploying Datasette on a Linux server :ref:`using systemd ` or to hosting providers :ref:`that support buildpacks `. (`#514 `__, `#997 `__) + +Other improvements in this release: -- Column action menu now shows the column type. (`#993 `__) -- Column action sort links now correctly link to the first page of sorted results. (`#989 `__) - :ref:`publish_cloud_run` documentation now covers Google Cloud SDK options. Thanks, Geoffrey Hing. (`#995 `__) - -.. _v0_50_a0: - -0.50a0 (2020-10-01) -------------------- - -- New column action menu - table columns now show a cog icon which provides a contextual menu for that column. (`#981 `__) - New ``datasette -o`` option which opens your browser as soon as Datasette starts up. (`#970 `__) -- ``sqlite3.enable_callback_tracebacks(True)`` so errors in custom SQL functions will now display tracebacks. (`#891 `__) +- Datasette now sets ``sqlite3.enable_callback_tracebacks(True)`` so that errors in custom SQL functions will display tracebacks. (`#891 `__) - Fixed two rendering bugs with column headers in portrait mobile view. (`#978 `__, `#980 `__) +- New ``db.table_column_details(table)`` introspection method for retrieving full details of the columns in a specific table, see :ref:`internals_database_introspection`. +- Fixed a routing bug with custom page wildcard templates. (`#996 `__) +- ``datasette publish heroku`` now deploys using Python 3.8.6. +- New ``datasette publish heroku --tar=`` option. (`#969 `__) +- ``OPTIONS`` requests against HTML pages no longer return a 500 error. (`#1001 `__) .. _v0_49_1: diff --git a/docs/internals.rst b/docs/internals.rst index 94c142c2..0fdd943c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -330,8 +330,8 @@ You can try out these messages (including the different visual styling of the th .. _internals_datasette_client: -.client -------- +datasette.client +---------------- Plugins can make internal HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins. @@ -365,7 +365,6 @@ It offers the following methods: For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. - .. _internals_database: Database class From 1bdbc8aa7f4fd7a768d456146e44da86cb1b36d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 10:57:55 -0700 Subject: [PATCH 0054/1866] Datasette now supports Python 3.9 --- docs/changelog.rst | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 046f5b4d..a6d74914 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ Other improvements in this release: - ``datasette publish heroku`` now deploys using Python 3.8.6. - New ``datasette publish heroku --tar=`` option. (`#969 `__) - ``OPTIONS`` requests against HTML pages no longer return a 500 error. (`#1001 `__) +- Datasette now supports Python 3.9. .. _v0_49_1: diff --git a/setup.py b/setup.py index 8443fb41..22d164b0 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,7 @@ setup( "Intended Audience :: End Users/Desktop", "Topic :: Database", "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.6", From ef76c9ea571eeefe136a18202f87ea8c4ef80ace Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 14:49:13 -0700 Subject: [PATCH 0055/1866] Link to annotated release notes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c65d17c..92c898af 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover ## News - * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. + * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. From 99488de329fa252f54db3166e46da468aa512388 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 14:50:19 -0700 Subject: [PATCH 0056/1866] Link to 0.50 annotated release notes --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a6d74914..f5758126 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,8 @@ Other improvements in this release: - ``OPTIONS`` requests against HTML pages no longer return a 500 error. (`#1001 `__) - Datasette now supports Python 3.9. +See also `Datasette 0.50: The annotated release notes `__. + .. _v0_49_1: 0.49.1 (2020-09-15) From 549a007683e38fd13da72be7b2f5ee1adb1484c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 16:13:41 -0700 Subject: [PATCH 0057/1866] Clarify that datasette.client HTTP calls are simulated --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 0fdd943c..a04de9fe 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -333,7 +333,7 @@ You can try out these messages (including the different visual styling of the th datasette.client ---------------- -Plugins can make internal HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins. +Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs. The ``datasette.client`` object is a wrapper around the `HTTPX Python library `__, providing an async-friendly API that is similar to the widely used `Requests library `__. From c13d184704a74654befe061500f55ca61f29ef1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 17:33:13 -0700 Subject: [PATCH 0058/1866] Emergency fix for broken links in 0.50, closes #1010 --- datasette/templates/row.html | 2 +- datasette/templates/table.html | 10 ++++----- tests/test_html.py | 41 ++++++++++++++++++++-------------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 6812b2d4..cd49a497 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -29,7 +29,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}

    +

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}

    {% include custom_table_templates %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index a5a3a180..ab2331c3 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -110,7 +110,7 @@

    ✎ View and edit SQL

    {% endif %} -

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}

    +

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}

    {% if suggested_facets %}

    @@ -159,10 +159,10 @@

    Advanced export

    JSON shape: - default, - array, - newline-delimited{% if primary_keys %}, - object + default, + array, + newline-delimited{% if primary_keys %}, + object {% endif %}

    diff --git a/tests/test_html.py b/tests/test_html.py index c0e3625e..aca4eedd 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -635,24 +635,24 @@ def test_table_csv_json_export_interface(app_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"].split("/")[-1] for l in links] + actual = [l["href"] for l in links] expected = [ - "simple_primary_key.json?id__gt=2", - "simple_primary_key.testall?id__gt=2", - "simple_primary_key.testnone?id__gt=2", - "simple_primary_key.testresponse?id__gt=2", - "simple_primary_key.csv?id__gt=2&_size=max", + "/fixtures/simple_primary_key.json?id__gt=2", + "/fixtures/simple_primary_key.testall?id__gt=2", + "/fixtures/simple_primary_key.testnone?id__gt=2", + "/fixtures/simple_primary_key.testresponse?id__gt=2", + "/fixtures/simple_primary_key.csv?id__gt=2&_size=max", "#export", ] assert expected == actual # And the advaced export box at the bottom: div = Soup(response.body, "html.parser").find("div", {"class": "advanced-export"}) - json_links = [a["href"].split("/")[-1] for a in div.find("p").findAll("a")] + json_links = [a["href"] for a in div.find("p").findAll("a")] assert [ - "simple_primary_key.json?id__gt=2", - "simple_primary_key.json?id__gt=2&_shape=array", - "simple_primary_key.json?id__gt=2&_shape=array&_nl=on", - "simple_primary_key.json?id__gt=2&_shape=object", + "/fixtures/simple_primary_key.json?id__gt=2", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=array", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on", + "/fixtures/simple_primary_key.json?id__gt=2&_shape=object", ] == json_links # And the CSV form form = div.find("form") @@ -666,6 +666,12 @@ def test_table_csv_json_export_interface(app_client): ] == inputs +def test_row_json_export_link(app_client): + response = app_client.get("/fixtures/simple_primary_key/1") + assert response.status == 200 + assert 'json' in response.text + + def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): response = app_client.get("/fixtures/facetable") assert response.status == 200 @@ -674,13 +680,13 @@ def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"].split("/")[-1] for l in links] + actual = [l["href"] for l in links] expected = [ - "facetable.json?_labels=on", - "facetable.testall?_labels=on", - "facetable.testnone?_labels=on", - "facetable.testresponse?_labels=on", - "facetable.csv?_labels=on&_size=max", + "/fixtures/facetable.json?_labels=on", + "/fixtures/facetable.testall?_labels=on", + "/fixtures/facetable.testnone?_labels=on", + "/fixtures/facetable.testresponse?_labels=on", + "/fixtures/facetable.csv?_labels=on&_size=max", "#export", ] assert expected == actual @@ -1347,6 +1353,7 @@ def test_metadata_sort_desc(app_client): assert list(reversed(expected)) == rows +@pytest.mark.xfail @pytest.mark.parametrize("base_url", ["/prefix/", "https://example.com/"]) @pytest.mark.parametrize( "path", From 9f6dd985bc0eff70f8a9ce65c6578bc43d2e172b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 17:39:45 -0700 Subject: [PATCH 0059/1866] Fix broken CSV/JSON export on query page, refs #1010 --- datasette/templates/query.html | 2 +- tests/test_html.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 0add74a8..c6574f31 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -58,7 +58,7 @@ {% if display_rows %} -

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}, CSV

    +

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}, CSV

    diff --git a/tests/test_html.py b/tests/test_html.py index aca4eedd..3f8cb178 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -672,6 +672,13 @@ def test_row_json_export_link(app_client): assert 'json' in response.text +def test_query_json_csv_export_links(app_client): + response = app_client.get("/fixtures?sql=select+1") + assert response.status == 200 + assert 'json' in response.text + assert 'CSV' in response.text + + def test_csv_json_export_links_include_labels_if_foreign_keys(app_client): response = app_client.get("/fixtures/facetable") assert response.status == 200 From 6fe30c348c58a0bc312552fd7a889731427b86e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 17:41:22 -0700 Subject: [PATCH 0060/1866] Release 0.50.1 Refs #1010 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f5758126..f0e825b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_50.1: + +0.50.1 (2020-10-09) +------------------- + +- Fixed a bug introduced in 0.50 where the export as JSON/CSV links on the table, row and query pages were broken. (`#1010 `__) + .. _v0_50: 0.50 (2020-10-09) From 7239175f63d150356a7f795cc4cabf7764d2cf68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 20:51:56 -0700 Subject: [PATCH 0061/1866] Fixed broken column header links, closes #1011 --- datasette/templates/_table.html | 4 ++-- tests/test_html.py | 10 +++++----- tests/test_plugins.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index 65789045..1dd94212 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -8,9 +8,9 @@ {{ column.name }} {% else %} {% if column.name == sort %} - {{ column.name }} ▼ + {{ column.name }} ▼ {% else %} - {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} + {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %} {% endif %} {% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 3f8cb178..5691b6c4 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -345,7 +345,7 @@ def test_sort_links(app_client): attrs_and_link_attrs = [ { "attrs": th.attrs, - "a_href": (th.find("a")["href"].split("/")[-1] if th.find("a") else None), + "a_href": (th.find("a")["href"] if th.find("a") else None), } for th in ths ] @@ -403,7 +403,7 @@ def test_sort_links(app_client): "data-column-not-null": "0", "data-is-pk": "0", }, - "a_href": "sortable?_sort_desc=sortable", + "a_href": "/fixtures/sortable?_sort_desc=sortable", }, { "attrs": { @@ -414,7 +414,7 @@ def test_sort_links(app_client): "data-column-not-null": "0", "data-is-pk": "0", }, - "a_href": "sortable?_sort=sortable_with_nulls", + "a_href": "/fixtures/sortable?_sort=sortable_with_nulls", }, { "attrs": { @@ -425,7 +425,7 @@ def test_sort_links(app_client): "data-column-not-null": "0", "data-is-pk": "0", }, - "a_href": "sortable?_sort=sortable_with_nulls_2", + "a_href": "/fixtures/sortable?_sort=sortable_with_nulls_2", }, { "attrs": { @@ -436,7 +436,7 @@ def test_sort_links(app_client): "data-column-not-null": "0", "data-is-pk": "0", }, - "a_href": "sortable?_sort=text", + "a_href": "/fixtures/sortable?_sort=text", }, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4b3634ab..08ed2e6b 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -497,9 +497,9 @@ def test_hook_register_output_renderer_can_render(app_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"].split("/")[-1] for l in links] + actual = [l["href"] for l in links] # Should not be present because we sent ?_no_can_render=1 - assert "facetable.testall?_labels=on" not in actual + assert "/fixtures/facetable.testall?_labels=on" not in actual # Check that it was passed the values we expected assert hasattr(app_client.ds, "_can_render_saw") assert { From 0e58ae7600212c075f5b8ae4b52d2af0e1acd4f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Oct 2020 20:53:47 -0700 Subject: [PATCH 0062/1866] Release 0.50.2 Refs #1011 --- docs/changelog.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f0e825b3..1d654485 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,15 @@ Changelog ========= -.. _v0_50.1: +.. _v0_50_2: + +0.50.2 (2020-10-09) +------------------- + +- Fixed another bug introduced in 0.50 where column header links on the table page were broken. (`#1011 `__) + + +.. _v0_50_1: 0.50.1 (2020-10-09) ------------------- From a67cb536f1fde4b3cf38032b61bcc6d38c30d762 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Oct 2020 13:54:27 -0700 Subject: [PATCH 0063/1866] Promote the Datasette Weekly newsletter --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 92c898af..66ddf803 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Live demo of current main: https://latest.datasette.io/ * Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) +Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. + ## News * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). From 822260fb30c9a6726a36975c9b8b26148bd66818 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Oct 2020 16:19:39 -0700 Subject: [PATCH 0064/1866] Improved homebrew instructions --- docs/installation.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 1a45c594..dcae738a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -27,12 +27,24 @@ Using Homebrew If you have a Mac and use `Homebrew `__, you can install Datasette by running this command in your terminal:: - brew install simonw/datasette/datasette + brew install datasette + +This should install the latest version. You can confirm by running:: + + datasette --version + +You can upgrade to the latest Homebrew packaged version using:: + + brew upgrade datasette Once you have installed Datasette you can install plugins using the following:: datasette install datasette-vega +If the latest packaged release of Datasette has not yet been made available through Homebrew, you can upgrade your Homebrew installation in-place using:: + + datasette install -U datasette + .. _installation_pip: Using pip From 7e7064385270dda09dc2aa396d290369a667a03f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Oct 2020 16:39:38 -0700 Subject: [PATCH 0065/1866] Removed --debug option, which didn't do anything - closes #814 --- README.md | 1 - datasette/cli.py | 6 +----- docs/changelog.rst | 1 - docs/datasette-serve-help.txt | 1 - tests/test_cli.py | 1 - 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index 66ddf803..8670936c 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,6 @@ Now visiting http://localhost:8001/History/downloads will show you a web interfa allowed. Use 0.0.0.0 to listen to all IPs and allow access from other machines. -p, --port INTEGER Port for server, defaults to 8001 - --debug Enable debug mode - useful for development --reload Automatically reload if database or code change detected - useful for development --cors Enable CORS by serving Access-Control-Allow- diff --git a/datasette/cli.py b/datasette/cli.py index 43e03f0a..55576013 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -286,9 +286,6 @@ def uninstall(packages, yes): default=8001, help="Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.", ) -@click.option( - "--debug", is_flag=True, help="Enable debug mode - useful for development" -) @click.option( "--reload", is_flag=True, @@ -366,7 +363,6 @@ def serve( immutable, host, port, - debug, reload, cors, sqlite_extensions, @@ -417,7 +413,7 @@ def serve( kwargs = dict( immutables=immutable, - cache_headers=not debug and not reload, + cache_headers=not reload, cors=cors, inspect_data=inspect_data, metadata=metadata_data, diff --git a/docs/changelog.rst b/docs/changelog.rst index 1d654485..3c56328c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,6 @@ Changelog - Fixed another bug introduced in 0.50 where column header links on the table page were broken. (`#1011 `__) - .. _v0_50_1: 0.50.1 (2020-10-09) diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index ac3ca49f..0457a321 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -14,7 +14,6 @@ Options: -p, --port INTEGER Port for server, defaults to 8001. Use -p 0 to automatically assign an available port. - --debug Enable debug mode - useful for development --reload Automatically reload if database or code change detected - useful for development diff --git a/tests/test_cli.py b/tests/test_cli.py index 09864602..0e1745c2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -89,7 +89,6 @@ def test_metadata_yaml(): immutable=[], host="127.0.0.1", port=8001, - debug=False, reload=False, cors=False, sqlite_extensions=[], From e34e84901d084ba3aaccecea020c5f9811865c8f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Oct 2020 17:18:45 -0700 Subject: [PATCH 0066/1866] Link: HTTP header pagination, closes #1014 --- datasette/renderer.py | 12 +++++++++++- docs/json_api.rst | 32 ++++++++++++++++++++++++++++++++ tests/test_api.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 27a5092f..bcde8516 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -5,6 +5,7 @@ from datasette.utils import ( CustomJSONEncoder, path_from_row_pks, ) +from datasette.utils.asgi import Response def convert_specific_columns_to_json(rows, columns, json_cols): @@ -44,6 +45,9 @@ def json_renderer(args, data, view_name): # Deal with the _shape option shape = args.get("_shape", "arrays") + + next_url = data.get("next_url") + if shape == "arrayfirst": data = [row[0] for row in data["rows"]] elif shape in ("objects", "object", "array"): @@ -71,6 +75,7 @@ def json_renderer(args, data, view_name): data = {"ok": False, "error": error} elif shape == "array": data = data["rows"] + elif shape == "arrays": pass else: @@ -89,4 +94,9 @@ def json_renderer(args, data, view_name): else: body = json.dumps(data, cls=CustomJSONEncoder) content_type = "application/json; charset=utf-8" - return {"body": body, "status_code": status_code, "content_type": content_type} + headers = {} + if next_url: + headers["link"] = '<{}>; rel="next"'.format(next_url) + return Response( + body, status=status_code, headers=headers, content_type=content_type + ) diff --git a/docs/json_api.rst b/docs/json_api.rst index af98eecd..8d45ac6f 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1,3 +1,5 @@ +.. _json_api: + JSON API ======== @@ -18,6 +20,8 @@ requests to fetch the data. If you start Datasette without the ``--cors`` option only JavaScript running on the same domain as Datasette will be able to access the API. +.. _json_api_shapes: + Different shapes ---------------- @@ -138,6 +142,34 @@ this format. The ``object`` keys are always strings. If your table has a compound primary key, the ``object`` keys will be a comma-separated string. +.. _json_api_pagination: + +Pagination +---------- + +The default JSON representation includes a ``"next_url"`` key which can be used to access the next page of results. If that key is null or missing then it means you have reached the final page of results. + +Other representations include pagination information in the ``link`` HTTP header. That header will look something like this:: + + link: ; rel="next" + +Here is an example Python function built using `requests `__ that returns a list of all of the paginated items from one of these API endpoints: + +.. code-block:: python + + def paginate(url): + items = [] + while url: + response = requests.get(url) + try: + url = response.links.get("next").get("url") + except AttributeError: + url = None + items.extend(response.json()) + return items + +.. _json_api_special: + Special JSON arguments ---------------------- diff --git a/tests/test_api.py b/tests/test_api.py index 4aa9811c..1d454ea1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1828,3 +1828,31 @@ def test_binary_data_in_json(app_client, path, expected_json, expected_text): assert response.json == expected_json else: assert response.text == expected_text + + +@pytest.mark.parametrize( + "qs", + [ + "", + "?_shape=arrays", + "?_shape=arrayfirst", + "?_shape=object", + "?_shape=objects", + "?_shape=array", + "?_shape=array&_nl=on", + ], +) +def test_paginate_using_link_header(app_client, qs): + path = "/fixtures/compound_three_primary_keys.json{}".format(qs) + num_pages = 0 + while path: + response = app_client.get(path) + num_pages += 1 + link = response.headers.get("link") + if link: + assert link.startswith("<") + assert link.endswith('>; rel="next"') + path = link[1:].split(">")[0] + else: + path = None + assert num_pages == 21 From acf07a67722aa74828744726187690b59d342494 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 11 Oct 2020 19:53:26 -0700 Subject: [PATCH 0067/1866] x button for clearing filters, refs #1016 --- datasette/static/table.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/datasette/static/table.js b/datasette/static/table.js index 7e839b9c..08c560d6 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -152,3 +152,33 @@ var DROPDOWN_ICON_SVG = ` el.querySelector('.filter-op') + ); + rows.forEach(row => { + var a = document.createElement('a'); + a.setAttribute('href', '#'); + a.setAttribute('aria-label', 'Remove this filter'); + a.style.textDecoration = 'none'; + a.innerText = x; + a.addEventListener('click', (ev) => { + ev.preventDefault(); + let row = ev.target.closest('div'); + row.querySelector('select').value = ''; + row.querySelector('.filter-op select').value = 'exact'; + row.querySelector('input.filter-value').value = ''; + ev.target.closest('a').style.display = 'none'; + }); + row.appendChild(a); + var column = row.querySelector('select'); + if (!column.value) { + a.style.display = 'none'; + } + }); +})(); From f3a087a578ae2c418103ad144b08c2fc8ad9c31d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Oct 2020 20:44:18 -0700 Subject: [PATCH 0068/1866] Edit SQL button on canned queries, closes #1019 --- datasette/static/app.css | 6 +++++ datasette/templates/query.html | 1 + datasette/views/database.py | 37 +++++++++++++++++++++++++--- tests/test_html.py | 45 ++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index da8ed2ab..d2494a34 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -447,3 +447,9 @@ svg.dropdown-menu-icon { border-right: 5px solid transparent; border-bottom: 5px solid #666; } + +.canned-query-edit-sql { + padding-left: 0.5em; + position: relative; + top: 1px; +} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index c6574f31..be180f33 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -54,6 +54,7 @@ {% if canned_write %}

    - home + home

    {{ super() }} {% endblock %} @@ -23,7 +23,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} -
    +

    Custom SQL query

    @@ -36,7 +36,7 @@ {% for table in tables %} {% if show_hidden or not table.hidden %}

    -

    {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

    +

    {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

    {% for column in table.columns[:9] %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}{% if table.columns|length > 9 %}...{% endif %}

    {% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

    @@ -44,14 +44,14 @@ {% endfor %} {% if hidden_count and not show_hidden %} -

    ... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

    +

    ... and {{ "{:,}".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}

    {% endif %} {% if views %}

    Views

    {% endif %} @@ -60,13 +60,13 @@

    Queries

    {% endif %} {% if allow_download %} -

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    +

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    {% endif %} {% include "_codemirror_foot.html" %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index c1adfc59..06e09635 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -10,7 +10,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% for database in databases %} -

    {{ database.name }}{% if database.private %} 🔒{% endif %}

    +

    {{ database.name }}{% if database.private %} 🔒{% endif %}

    {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%} {% if database.hidden_tables_count -%} @@ -21,8 +21,7 @@ {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %} {% endif %}

    -

    {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}

    +

    {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}

    {% endfor %} {% endblock %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index be180f33..911119bb 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -20,8 +20,8 @@ {% block nav %}

    - home / - {{ database }} + home / + {{ database }}

    {{ super() }} {% endblock %} @@ -32,7 +32,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}

    {% if not hide_sql %} {% if editable and allow_execute_sql %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index cd49a497..916980b6 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -17,9 +17,9 @@ {% block nav %}

    - home / - {{ database }} / - {{ table }} + home / + {{ database }} / + {{ table }}

    {{ super() }} {% endblock %} @@ -38,7 +38,7 @@
      {% for other in foreign_key_tables %}
    • - + {{ "{:,}".format(other.count) }} row{% if other.count == 1 %}{% else %}s{% endif %} from {{ other.other_column }} in {{ other.other_table }}
      • diff --git a/datasette/templates/table.html b/datasette/templates/table.html index ab2331c3..3f8c2fee 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -5,7 +5,7 @@ {% block extra_head %} {{ super() }} - + + - -
      + + +
      +
      + +
      + + +
      + +
      + + + +

      Pattern Portfolio

      -
      -

      .hd for /database/table/row

      - -

      Messages

      -
      -

      Example message

      -

      Example message

      -

      Example message

      -
      + + + + + +

      Header for /database/table/row and Messages

      + +
      + +
      + +

      Example message

      +

      Example message

      +

      Example message

      + + + + + + + + + +

      .bd for /

      -
      +

      Datasette Fixtures

      An example SQLite database demonstrating Datasette @@ -60,9 +88,16 @@ 6 rows in 2 tables

      names, foo

      -
      +
      + + + + + + +

      .bd for /database

      -
      +

      fixtures

      Test tables description @@ -102,9 +137,15 @@

      pk, name

      2 rows

      -
      + + + + + + +

      .bd for /database/table

      -
      +

      roadside_attraction_characteristics

      Data license: @@ -203,9 +244,109 @@

      - -

      ✎ View and edit SQL

      -

      This data as json, CSV (advanced)

      + + + +
      +

      2 extra where clauses

      + +
      + + +

      ✎ View and edit SQL

      + +

      This data as json, CSV (advanced)

      + +

      + Suggested facets: tags, created (date), tags (array) +

      + + + + + + +
      + +
      +

      + tags (array) + + ✖ + +

      + +
      + +
      +

      + created + + ✖ + +

      + +
      + +
      +

      + city_id + + ✖ + +

      + +
      + +
      +
    @@ -266,9 +407,20 @@ attraction_id INTEGER REFERENCES roadside_attractions(pk), characteristic_id INTEGER REFERENCES attraction_characteristic(pk) ); - + + + + + + + + + + + +

    .bd for /database/table/row

    -
    +

    roadside_attractions: 2

    This data as json

    @@ -309,9 +461,21 @@ from attraction_id in roadside_attraction_characteristics - + + + + + + + + + + + +

    .ft

    -
    Powered by Datasette + +
    + From 6dff22eff8a52253a6c2bdf3e32f082fbf81b921 Mon Sep 17 00:00:00 2001 From: Natalie Downe Date: Tue, 27 Oct 2020 11:39:35 -0700 Subject: [PATCH 0106/1866] Visited link colours --- datasette/static/app.css | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 87ec5f01..1ad04618 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -164,11 +164,14 @@ footer, box-sizing: border-box; } -a:link, -a:visited { +a:link { color: #276890; text-decoration: underline; } +a:visited { + color: #54AC8E; + text-decoration: underline; +} a:hover, a:focus, a:active { @@ -363,10 +366,6 @@ th { } table a:link { text-decoration: none; - color: #445ac8; -} -table a:visited { - color: #8f54c4; } .rows-and-columns td:before { display: block; From df19a48a3b72a51feb4203c44903451cc9e6c1bf Mon Sep 17 00:00:00 2001 From: Natalie Downe Date: Tue, 27 Oct 2020 11:40:08 -0700 Subject: [PATCH 0107/1866] Implemented new Natalie design --- datasette/static/app.css | 7 +++++-- datasette/templates/base.html | 10 +++++----- datasette/templates/database.html | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1ad04618..dff882af 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -210,7 +210,7 @@ ol.spaced { margin-bottom: 0.8rem; } ul.bullets { - padding-left: 0.9rem; + padding-left: 1.25rem; } ul.bullets li, ul.spaced li, @@ -290,7 +290,9 @@ section.content { } /* Footer */ - +footer { + margin-top: 1rem; +} /* Components ============================================================== */ @@ -568,6 +570,7 @@ form button[type=button] { width: auto; display: inline-block; box-shadow: 1px 2px 8px 2px rgba(0,0,0,0.08); + background-color: white; } .download-sqlite em { diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 84708325..03de2115 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -14,7 +14,7 @@ - -
    +
    {% block messages %} {% if show_messages %} {% for message, message_type in show_messages() %} @@ -37,9 +37,9 @@ {% block content %} {% endblock %} -
    + -
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    +
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    {% for body_script in body_scripts %} diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 2f844b6a..3b89d68b 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -49,7 +49,7 @@ {% if views %}

    Views

    -
      +
        {% for view in views %}
      • {{ view.name }}{% if view.private %} 🔒{% endif %}
        • {% endfor %} @@ -58,7 +58,7 @@ {% if queries %}

        Queries

        -
          +
            {% for query in queries %}
          • {{ query.title or query.name }}{% if query.private %} 🔒{% endif %}
            • {% endfor %} From fe5e813f068abd2ee63994b2baf530c7abe34de1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 11:57:34 -0700 Subject: [PATCH 0108/1866] Styled facets with different bullets --- datasette/static/app.css | 15 +++++++++++++-- datasette/templates/table.html | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index dff882af..2dfc6b15 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -55,7 +55,12 @@ th { padding-right: 1em; white-space: nowrap; } - +strong { + font-weight: bold; +} +em { + font-style: italic; +} /* end reset */ @@ -205,11 +210,13 @@ pre { } ul.bullets, +ul.tight-bullets, ul.spaced, ol.spaced { margin-bottom: 0.8rem; } -ul.bullets { +ul.bullets, +ul.tight-bullets { padding-left: 1.25rem; } ul.bullets li, @@ -220,6 +227,10 @@ ol.spaced li { ul.bullets li { list-style-type: circle; } +ul.tight-bullets li { + list-style-type: disc; + margin-bottom: 0; +} a.not-underlined { text-decoration: none; } diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 3f8c2fee..bc8cfc0a 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -132,7 +132,7 @@ ✖ {% endif %}

            -
              +
                {% for facet_value in facet_info.results %} {% if not facet_value.selected %}
              • {{ (facet_value.label if facet_value.label is not none else "_") }} {{ "{:,}".format(facet_value.count) }}
                • From 62286b46a9b434467ab7dee37ec2f8619ca0d1b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:01:44 -0700 Subject: [PATCH 0109/1866] Tighten up table column CSS --- datasette/templates/table.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index bc8cfc0a..6c27beee 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -8,9 +8,9 @@ {% endblock %} From dab4b73f7d76f43e67ae1e2b74921f62db71925c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:07:25 -0700 Subject: [PATCH 0110/1866] White cards on mobile --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 2dfc6b15..c89f412b 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -654,6 +654,7 @@ p.zero-results { border-bottom: 1px solid #eee; padding: 0; padding-left: 10%; + background-color: white; } .rows-and-columns td:before { From f49d15a7583fafb94e7a7fcfe504d333812139f3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:20:28 -0700 Subject: [PATCH 0111/1866] word-break: break-word; --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index c89f412b..7988252e 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -119,7 +119,8 @@ h6, font-weight: 700; font-size: 1rem; margin: 0; - padding: 0 + padding: 0; + word-break: break-word; } h1, .header1 { From c069d481af736f43e82598752f30ddc98bcb4b29 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:27:14 -0700 Subject: [PATCH 0112/1866] Mobile view cards now have rounded corners --- datasette/static/app.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 7988252e..adbfccab 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -647,6 +647,9 @@ p.zero-results { .rows-and-columns tr { border: 1px solid #ccc; margin-bottom: 1em; + border-radius: 10px; + background-color: white; + padding: 0.2rem; } .rows-and-columns td { @@ -655,7 +658,6 @@ p.zero-results { border-bottom: 1px solid #eee; padding: 0; padding-left: 10%; - background-color: white; } .rows-and-columns td:before { From 18977ce8026e71bc29c51eef2b46dbaa288042d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:28:50 -0700 Subject: [PATCH 0113/1866] Off-white yellow is now off-white blue --- datasette/static/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index adbfccab..085d829c 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -73,7 +73,7 @@ body { line-height: 1.5; color: #111A35; text-align: left; - background-color: #fefdf4; + background-color: #F8FAFB; } /* Helper Styles ===========================================================*/ From e7dd3434e1f3f20129798bcea1a629717eec1649 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:30:40 -0700 Subject: [PATCH 0114/1866] No underline on nav links in header --- datasette/static/app.css | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 085d829c..b6dfd7f3 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -272,7 +272,9 @@ footer a:visited, footer a:hover, footer a:focus, footer a:active, -footer button.button-as-link, +footer button.button-as-link { + color: rgba(255,255,244,0.8); +} header a:link, header a:visited, header a:hover, @@ -280,6 +282,7 @@ header a:focus, header a:active, header button.button-as-link { color: rgba(255,255,244,0.8); + text-decoration: none; } footer a:hover, From e5f5034bcdc71e4bc62a6a155ca60eb41910c335 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 12:34:35 -0700 Subject: [PATCH 0115/1866] Fixed broken footer test --- tests/test_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_html.py b/tests/test_html.py index 3af9816f..06b11de5 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1058,7 +1058,7 @@ def assert_querystring_equal(expected, actual): def assert_footer_links(soup): - footer_links = soup.find("div", {"class": "ft"}).findAll("a") + footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) datasette_link, license_link, source_link, about_link = footer_links assert "Datasette" == datasette_link.text.strip() From c3aba4aa986fdba39705a35de02d446db80a26b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 13:39:07 -0700 Subject: [PATCH 0116/1866] --cors for /name.db downloads, refs #1057 --- datasette/utils/asgi.py | 17 +++++++++++++---- datasette/views/database.py | 4 ++++ tests/fixtures.py | 2 +- tests/test_api.py | 1 + 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 911038ab..bd388390 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -247,9 +247,9 @@ async def asgi_start(send, status, headers=None, content_type="text/plain"): async def asgi_send_file( - send, filepath, filename=None, content_type=None, chunk_size=4096 + send, filepath, filename=None, content_type=None, chunk_size=4096, headers=None ): - headers = {} + headers = headers or {} if filename: headers["content-disposition"] = 'attachment; filename="{}"'.format(filename) first = True @@ -395,13 +395,22 @@ class Response: class AsgiFileDownload: def __init__( - self, filepath, filename=None, content_type="application/octet-stream" + self, + filepath, + filename=None, + content_type="application/octet-stream", + headers=None, ): + self.headers = headers or {} self.filepath = filepath self.filename = filename self.content_type = content_type async def asgi_send(self, send): return await asgi_send_file( - send, self.filepath, filename=self.filename, content_type=self.content_type + send, + self.filepath, + filename=self.filename, + content_type=self.content_type, + headers=self.headers, ) diff --git a/datasette/views/database.py b/datasette/views/database.py index 74509278..025e853d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -144,10 +144,14 @@ class DatabaseDownload(DataView): if not db.path: raise DatasetteError("Cannot download database", status=404) filepath = db.path + headers = {} + if self.ds.cors: + headers["Access-Control-Allow-Origin"] = "*" return AsgiFileDownload( filepath, filename=os.path.basename(filepath), content_type="application/octet-stream", + headers=headers, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index d8c92561..7786ca8c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -227,7 +227,7 @@ def app_client_with_dot(): @pytest.fixture(scope="session") def app_client_with_cors(): - with make_app_client(cors=True) as client: + with make_app_client(is_immutable=True, cors=True) as client: yield client diff --git a/tests/test_api.py b/tests/test_api.py index 1d454ea1..461d3f81 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1739,6 +1739,7 @@ def test_trace(app_client): @pytest.mark.parametrize( "path,status_code", [ + ("/fixtures.db", 200), ("/fixtures.json", 200), ("/fixtures/no_primary_key.json", 200), # A 400 invalid SQL query should still have the header: From 7d9fedc176717a7e3d22a96575ae0aada5a65440 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Oct 2020 20:15:41 -0700 Subject: [PATCH 0117/1866] Cascading permissions for .db download, closes #1058 --- datasette/views/database.py | 11 ++++++++--- tests/plugins/my_plugin.py | 2 ++ tests/test_permissions.py | 12 ++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 025e853d..00fbc0b0 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -131,9 +131,14 @@ class DatabaseDownload(DataView): name = "database_download" async def view_get(self, request, database, hash, correct_hash_present, **kwargs): - await self.check_permission(request, "view-instance") - await self.check_permission(request, "view-database", database) - await self.check_permission(request, "view-database-download", database) + await self.check_permissions( + request, + [ + ("view-database-download", database), + ("view-database", database), + "view-instance", + ], + ) if database not in self.ds.databases: raise DatasetteError("Invalid database", status=404) db = self.ds.databases[database] diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 0870eb19..0dd0ad26 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -193,6 +193,8 @@ def permission_allowed(actor, action): return True elif action == "this_is_denied": return False + elif action == "view-database-download": + return (actor and actor.get("can_download")) or None @hookimpl diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 3c11985c..a935a495 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -394,7 +394,7 @@ def test_view_instance(path, view_instance_client): @pytest.fixture(scope="session") def cascade_app_client(): - with make_app_client() as client: + with make_app_client(is_immutable=True) as client: yield client @@ -439,6 +439,11 @@ def cascade_app_client(): ("/fixtures", [], 403), ("/fixtures", ["instance"], 403), ("/fixtures", ["database"], 200), + # Downloading the fixtures.db file + ("/fixtures.db", [], 403), + ("/fixtures.db", ["instance"], 403), + ("/fixtures.db", ["database"], 200), + ("/fixtures.db", ["download"], 200), ], ) def test_permissions_cascade(cascade_app_client, path, permissions, expected_status): @@ -447,6 +452,9 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta deny = {} previous_metadata = cascade_app_client.ds._metadata updated_metadata = copy.deepcopy(previous_metadata) + actor = {"id": "test"} + if "download" in permissions: + actor["can_download"] = 1 try: # Set up the different allow blocks updated_metadata["allow"] = allow if "instance" in permissions else deny @@ -462,7 +470,7 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta cascade_app_client.ds._metadata = updated_metadata response = cascade_app_client.get( path, - cookies={"ds_actor": cascade_app_client.actor_cookie({"id": "test"})}, + cookies={"ds_actor": cascade_app_client.actor_cookie(actor)}, ) assert expected_status == response.status finally: From 879617265262024edd93722adcdcb6c21e57f5f7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2020 10:08:27 -0700 Subject: [PATCH 0118/1866] Update aiofiles requirement from <0.6,>=0.4 to >=0.4,<0.7 (#1059) Updates the requirements on [aiofiles](https://github.com/Tinche/aiofiles) to permit the latest version. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.4.0...v0.6.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0de7f92d..c4dea142 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", - "aiofiles>=0.4,<0.6", + "aiofiles>=0.4,<0.7", "janus>=0.4,<0.7", "asgi-csrf>=0.6", "PyYAML~=5.3", From abcf0222496d8148b2e585ffa0ff192270a04b06 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Oct 2020 10:11:07 -0700 Subject: [PATCH 0119/1866] Margin bottom on metadata description --- datasette/static/app.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index b6dfd7f3..8b462b35 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -81,6 +81,9 @@ body { .intro { font-size: 1rem; } +.metadata-description { + margin-bottom: 1em; +} p { margin: 0 0 0.75rem 0; padding: 0; From cefd058c1c216a184bb63c79abba66893977c18e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Oct 2020 20:38:15 -0700 Subject: [PATCH 0120/1866] New explicit versioning mechanism Closes #1054 --- .gitattributes | 1 - datasette/_version.py | 556 ------------ datasette/cli.py | 3 +- datasette/version.py | 6 +- setup.cfg | 8 - setup.py | 6 +- tests/test_api.py | 2 + tests/test_cli.py | 7 + versioneer.py | 1885 ----------------------------------------- 9 files changed, 14 insertions(+), 2460 deletions(-) delete mode 100644 datasette/_version.py delete mode 100644 versioneer.py diff --git a/.gitattributes b/.gitattributes index e5e5865f..744258eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ -datasette/_version.py export-subst datasette/static/codemirror-* linguist-vendored diff --git a/datasette/_version.py b/datasette/_version.py deleted file mode 100644 index 5783f30f..00000000 --- a/datasette/_version.py +++ /dev/null @@ -1,556 +0,0 @@ -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "datasette-" - cfg.versionfile_source = "datasette/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/datasette/cli.py b/datasette/cli.py index ece03636..04d2950b 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -25,6 +25,7 @@ from .utils import ( ValueAsBooleanError, ) from .utils.testing import TestClient +from .version import __version__ class Config(click.ParamType): @@ -65,7 +66,7 @@ class Config(click.ParamType): @click.group(cls=DefaultGroup, default="serve", default_if_no_args=True) -@click.version_option() +@click.version_option(version=__version__) def cli(): """ Datasette! diff --git a/datasette/version.py b/datasette/version.py index e1fed2c4..b57d7a12 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,6 +1,2 @@ -from ._version import get_versions - -__version__ = get_versions()["version"] -del get_versions - +__version__ = "0.51.a0" __version_info__ = tuple(__version__.split(".")) diff --git a/setup.cfg b/setup.cfg index 1617b3eb..ebf43062 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,13 +1,5 @@ [aliases] test=pytest -[versioneer] -VCS = git -style = pep440 -versionfile_source = datasette/_version.py -versionfile_build = datasette/_version.py -tag_prefix = -parentdir_prefix = datasette- - [flake8] max-line-length = 160 diff --git a/setup.py b/setup.py index c4dea142..82696b38 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ +from re import VERBOSE from setuptools import setup, find_packages import os import sys -import versioneer - def get_long_description(): with open( @@ -24,8 +23,7 @@ def get_version(): setup( name="datasette", - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=get_version(), description="An open source multi-tool for exploring and publishing data", long_description=get_long_description(), long_description_content_type="text/markdown", diff --git a/tests/test_api.py b/tests/test_api.py index 461d3f81..53f33a9c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 +from datasette.version import __version__ from .fixtures import ( # noqa app_client, app_client_no_files, @@ -1290,6 +1291,7 @@ def test_versions_json(app_client): assert "full" in response.json["python"] assert "datasette" in response.json assert "version" in response.json["datasette"] + assert response.json["datasette"]["version"] == __version__ assert "sqlite" in response.json assert "version" in response.json["sqlite"] assert "fts_versions" in response.json["sqlite"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 1aff8cd1..b27cd5a8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,6 +6,7 @@ from .fixtures import ( ) from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve +from datasette.version import __version__ from click.testing import CliRunner import io import json @@ -156,3 +157,9 @@ def test_uninstall(run_module): runner.invoke(cli, ["uninstall", "datasette-mock-plugin", "-y"]) run_module.assert_called_once_with("pip", run_name="__main__") assert sys.argv == ["pip", "uninstall", "datasette-mock-plugin", "-y"] + + +def test_version(): + runner = CliRunner() + result = runner.invoke(cli, ["--version"]) + assert result.output == "cli, version {}\n".format(__version__) diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index 858fc0bd..00000000 --- a/versioneer.py +++ /dev/null @@ -1,1885 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other languages) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From 89519f9a3765bce7544d83d872db987c75757d9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Oct 2020 21:05:40 -0700 Subject: [PATCH 0121/1866] Fixed bug with download of BLOB null, refs #1050 --- datasette/views/table.py | 2 +- tests/fixtures.py | 1 + tests/test_api.py | 6 ++++-- tests/test_html.py | 29 ++++++++++++++++++++++------- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 717341ae..d190b6af 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1083,7 +1083,7 @@ class BlobView(BaseView): "Content-Disposition": 'attachment; filename="{}"'.format(filename), } return Response( - body=rows[0][column], + body=rows[0][column] or b"", status=200, headers=headers, content_type="application/binary", diff --git a/tests/fixtures.py b/tests/fixtures.py index 7786ca8c..31638fc8 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -667,6 +667,7 @@ CREATE VIEW searchable_view_configured_by_metadata AS TABLE_PARAMETERIZED_SQL = [ ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (null);", []), ] EXTRA_DATABASE_SQL = """ diff --git a/tests/test_api.py b/tests/test_api.py index 53f33a9c..5e9c1a0a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -106,7 +106,7 @@ def test_database_page(app_client): "name": "binary_data", "columns": ["data"], "primary_keys": [], - "count": 2, + "count": 3, "hidden": False, "fts_table": None, "foreign_keys": {"incoming": [], "outgoing": []}, @@ -1812,6 +1812,7 @@ def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file): [ {"rowid": 1, "data": {"$base64": True, "encoded": "FRwCx60F/g=="}}, {"rowid": 2, "data": {"$base64": True, "encoded": "FRwDx60F/g=="}}, + {"rowid": 3, "data": None}, ], None, ), @@ -1820,7 +1821,8 @@ def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file): None, ( '{"rowid": 1, "data": {"$base64": true, "encoded": "FRwCx60F/g=="}}\n' - '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}' + '{"rowid": 2, "data": {"$base64": true, "encoded": "FRwDx60F/g=="}}\n' + '{"rowid": 3, "data": null}' ), ), ], diff --git a/tests/test_html.py b/tests/test_html.py index 06b11de5..3c1101d2 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1238,21 +1238,36 @@ def test_binary_data_display(app_client): '
    ', '', ], + [ + '', + '', + '', + ], ] assert expected_tds == [ [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") ] -def test_blob_download(app_client): - response = app_client.get("/fixtures/binary_data/-/blob/1/data.blob") +@pytest.mark.parametrize( + "path,expected_body,expected_filename", + [ + ( + "/fixtures/binary_data/-/blob/1/data.blob", + b"\x15\x1c\x02\xc7\xad\x05\xfe", + "binary_data-1-data.blob", + ), + ("/fixtures/binary_data/-/blob/3/data.blob", b"", "binary_data-3-data.blob"), + ], +) +def test_blob_download(app_client, path, expected_body, expected_filename): + response = app_client.get(path) assert response.status == 200 - assert response.body == b"\x15\x1c\x02\xc7\xad\x05\xfe" + assert response.body == expected_body assert response.headers["x-content-type-options"] == "nosniff" - assert ( - response.headers["content-disposition"] - == 'attachment; filename="binary_data-1-data.blob"' - ) + assert response.headers[ + "content-disposition" + ] == 'attachment; filename="{}"'.format(expected_filename) assert response.headers["content-type"] == "application/binary" From d6f9ff71378c4eab34dad181c23cfc143a4aef2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 12:35:25 -0700 Subject: [PATCH 0122/1866] Docs on Designing URLs for your plugin - closes #1053 --- docs/introspection.rst | 2 ++ docs/plugin_hooks.rst | 2 ++ docs/writing_plugins.rst | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/docs/introspection.rst b/docs/introspection.rst index 08006529..698ba95f 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -1,3 +1,5 @@ +.. _introspection: + Introspection ============= diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 5cfae22a..b2c62ccd 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -556,6 +556,8 @@ The view function can be a regular function or an ``async def`` function, depend The function can either return a :ref:`internals_response` or it can return nothing and instead respond directly to the request using the ASGI ``send`` function (for advanced uses only). +See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin. + Examples: `datasette-auth-github `__, `datasette-psutil `__ .. _plugin_register_facet_classes: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index f763f617..29fcca13 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -202,6 +202,36 @@ The plugin configuration could also be set at the top level of ``metadata.json`` Now that ``datasette-cluster-map`` plugin configuration will apply to every table in every database. +.. _writing_plugins_designing_urls: + +Designing URLs for your plugin +------------------------------ + +You can register new URL routes within Datasette using the :ref:`plugin_register_routes` plugin hook. + +Datasette's default URLs include these: + +- ``/dbname`` - database page +- ``/dbname/tablename`` - table page +- ``/dbname/tablename/pk`` - row page + +See :ref:`pages` and :ref:`introspection` for more default URL routes. + +To avoid accidentally conflicting with a database file that may be loaded into Datasette, plugins should register URLs using a ``/-/`` prefix. For example, if your plugin adds a new interface for uploading Excel files you might register a URL route like this one: + +- ``/-/upload-excel`` + +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic `__ on GitHub. + +If your plugin includes functionality that relates to a specific database you could also register a URL route like this: + +- ``/dbname/-/upload-excel`` + +Reserving routes under ``/dbname/tablename/-/...`` is not a good idea because a table could conceivably include a row with a primary key value of ``-``. Instead, you could use a pattern like this: + +- ``/dbname/-/upload-excel/tablename`` + + .. _writing_plugins_building_urls: Building URLs within plugins From 78b3eeaad9189eb737014f53212082684f4bb0d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 15:01:38 -0700 Subject: [PATCH 0123/1866] .blob output renderer * _blob_hash= checking plus refactored to use new BadRequest class, refs #1050 * Replace BlobView with new .blob renderer, closes #1050 * .blob downloads on arbitrary queries, closes #1051 --- datasette/app.py | 19 +++++------ datasette/blob_renderer.py | 61 ++++++++++++++++++++++++++++++++++ datasette/plugins.py | 1 + datasette/templates/query.html | 2 +- datasette/utils/asgi.py | 17 +++++++--- datasette/views/base.py | 5 +-- datasette/views/database.py | 20 +++++++++++ datasette/views/table.py | 61 ++++------------------------------ docs/pages.rst | 11 ------ tests/test_csv.py | 18 ++++++++++ tests/test_html.py | 60 ++++++++++++++++++--------------- tests/test_permissions.py | 11 ------ 12 files changed, 165 insertions(+), 121 deletions(-) create mode 100644 datasette/blob_renderer.py diff --git a/datasette/app.py b/datasette/app.py index 5b50294f..3016043a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -38,7 +38,7 @@ from .views.special import ( PermissionsDebugView, MessagesDebugView, ) -from .views.table import RowView, TableView, BlobView +from .views.table import RowView, TableView from .renderer import json_renderer from .database import Database, QueryInterrupted @@ -60,6 +60,7 @@ from .utils import ( ) from .utils.asgi import ( AsgiLifespan, + Base400, Forbidden, NotFound, Request, @@ -923,10 +924,6 @@ class Datasette: + renderer_regex + r")?$", ) - add_route( - BlobView.as_view(self), - r"/(?P[^/]+)/(?P
    2<Binary:\xa07\xa0bytes>33\xa0
    [^/]+?)/\-/blob/(?P[^/]+?)/(?P[^/]+)\.blob$", - ) self._register_custom_units() async def setup_db(): @@ -1113,11 +1110,7 @@ class DatasetteRouter: pdb.post_mortem(exception.__traceback__) title = None - if isinstance(exception, NotFound): - status = 404 - info = {} - message = exception.args[0] - elif isinstance(exception, Forbidden): + if isinstance(exception, Forbidden): status = 403 info = {} message = exception.args[0] @@ -1129,6 +1122,10 @@ class DatasetteRouter: if custom_response is not None: await custom_response.asgi_send(send) return + elif isinstance(exception, Base400): + status = exception.status + info = {} + message = exception.args[0] elif isinstance(exception, DatasetteError): status = exception.status info = exception.error_dict @@ -1308,6 +1305,6 @@ class Urls: return "{}/{}".format(self.table(database, table), row_path) def row_blob(self, database, table, row_path, column): - return self.table(database, table) + "/-/blob/{}/{}.blob".format( + return self.table(database, table) + "/{}.blob?_blob_column={}".format( row_path, urllib.parse.quote_plus(column) ) diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py new file mode 100644 index 00000000..794b153e --- /dev/null +++ b/datasette/blob_renderer.py @@ -0,0 +1,61 @@ +from datasette import hookimpl +from datasette.utils.asgi import Response, BadRequest +from datasette.utils import to_css_class +import hashlib + +_BLOB_COLUMN = "_blob_column" +_BLOB_HASH = "_blob_hash" + + +async def render_blob(datasette, database, rows, columns, request, table, view_name): + if _BLOB_COLUMN not in request.args: + raise BadRequest("?{}= is required".format(_BLOB_COLUMN)) + blob_column = request.args[_BLOB_COLUMN] + if blob_column not in columns: + raise BadRequest("{} is not a valid column".format(blob_column)) + + # If ?_blob_hash= provided, use that to select the row - otherwise use first row + blob_hash = None + if _BLOB_HASH in request.args: + blob_hash = request.args[_BLOB_HASH] + for row in rows: + value = row[blob_column] + if hashlib.sha256(value).hexdigest() == blob_hash: + break + else: + # Loop did not break + raise BadRequest( + "Link has expired - the requested binary content has changed or could not be found." + ) + else: + row = rows[0] + + value = row[blob_column] + filename_bits = [] + if table: + filename_bits.append(to_css_class(table)) + if "pk_path" in request.url_vars: + filename_bits.append(request.url_vars["pk_path"]) + filename_bits.append(to_css_class(blob_column)) + if blob_hash: + filename_bits.append(blob_hash[:6]) + filename = "-".join(filename_bits) + ".blob" + headers = { + "X-Content-Type-Options": "nosniff", + "Content-Disposition": 'attachment; filename="{}"'.format(filename), + } + return Response( + body=value or b"", + status=200, + headers=headers, + content_type="application/binary", + ) + + +@hookimpl +def register_output_renderer(): + return { + "extension": "blob", + "render": render_blob, + "can_render": lambda: False, + } diff --git a/datasette/plugins.py b/datasette/plugins.py index cb3d2c34..1c2f392f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -12,6 +12,7 @@ DEFAULT_PLUGINS = ( "datasette.actor_auth_cookie", "datasette.default_permissions", "datasette.default_magic_parameters", + "datasette.blob_renderer", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 78a1c123..9b3fff25 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -70,7 +70,7 @@ {% for row in display_rows %} {% for column, td in zip(columns, row) %} - + {% endfor %} {% endfor %} diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index bd388390..f438f829 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,4 +1,5 @@ import json +from os import EX_CANTCREAT from datasette.utils import MultiParams from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl @@ -15,12 +16,20 @@ Morsel._reserved["samesite"] = "SameSite" # https://github.com/encode/starlette/blob/519f575/starlette/responses.py#L17 -class NotFound(Exception): - pass +class Base400(Exception): + status = 400 -class Forbidden(Exception): - pass +class NotFound(Base400): + status = 404 + + +class Forbidden(Base400): + status = 403 + + +class BadRequest(Base400): + status = 400 SAMESITE_VALUES = ("strict", "lax", "none") diff --git a/datasette/views/base.py b/datasette/views/base.py index f9bbe45d..4432ddca 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -26,6 +26,7 @@ from datasette.utils.asgi import ( Forbidden, NotFound, Response, + BadRequest, ) ureg = pint.UnitRegistry() @@ -260,9 +261,9 @@ class DataView(BaseView): if stream: # Some quick sanity checks if not self.ds.config("allow_csv_stream"): - raise DatasetteError("CSV streaming is disabled", status=400) + raise BadRequest("CSV streaming is disabled") if request.args.get("_next"): - raise DatasetteError("_next not allowed for CSV streaming", status=400) + raise BadRequest("_next not allowed for CSV streaming") kwargs["_size"] = "max" # Fetch the first page try: diff --git a/datasette/views/database.py b/datasette/views/database.py index 00fbc0b0..8b9e8833 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,5 @@ import os +import hashlib import itertools import jinja2 import json @@ -10,6 +11,7 @@ from datasette.utils import ( validate_sql_select, is_url, path_with_added_args, + path_with_format, path_with_removed_args, InvalidSql, ) @@ -342,6 +344,24 @@ class QueryView(DataView): url=jinja2.escape(value.strip()) ) ) + elif isinstance(display_value, bytes): + blob_url = path_with_format( + request, + "blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256( + display_value + ).hexdigest(), + }, + ) + display_value = jinja2.Markup( + '<Binary: {} byte{}>'.format( + blob_url, + len(display_value), + "" if len(value) == 1 else "s", + ) + ) display_row.append(display_value) display_rows.append(display_row) diff --git a/datasette/views/table.py b/datasette/views/table.py index d190b6af..079e0b0a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -23,9 +23,9 @@ from datasette.utils import ( urlsafe_components, value_as_boolean, ) -from datasette.utils.asgi import NotFound, Response +from datasette.utils.asgi import BadRequest, NotFound from datasette.filters import Filters -from .base import BaseView, DataView, DatasetteError, ureg +from .base import DataView, DatasetteError, ureg from .database import QueryView LINK_WITH_LABEL = ( @@ -469,7 +469,7 @@ class TableView(RowTableShared): for i, (key, search_text) in enumerate(search_args.items()): search_col = key.split("_search_", 1)[1] if search_col not in await db.table_columns(fts_table): - raise DatasetteError("Cannot search by that column", status=400) + raise BadRequest("Cannot search by that column") where_clauses.append( "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( @@ -614,11 +614,11 @@ class TableView(RowTableShared): raise ValueError except ValueError: - raise DatasetteError("_size must be a positive integer", status=400) + raise BadRequest("_size must be a positive integer") if page_size > self.ds.max_returned_rows: - raise DatasetteError( - "_size must be <= {}".format(self.ds.max_returned_rows), status=400 + raise BadRequest( + "_size must be <= {}".format(self.ds.max_returned_rows) ) extra_args["page_size"] = page_size @@ -665,7 +665,7 @@ class TableView(RowTableShared): if not self.ds.config("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): - raise DatasetteError("_facet= is not allowed", status=400) + raise BadRequest("_facet= is not allowed") # pylint: disable=no-member facet_classes = list( @@ -1041,50 +1041,3 @@ class RowView(RowTableShared): ) foreign_key_tables.append({**fk, **{"count": count}}) return foreign_key_tables - - -class BlobView(BaseView): - async def get(self, request, db_name, table, pk_path, column): - await self.check_permissions( - request, - [ - ("view-table", (db_name, table)), - ("view-database", db_name), - "view-instance", - ], - ) - try: - db = self.ds.get_database(db_name) - except KeyError: - raise NotFound("Database {} does not exist".format(db_name)) - if not await db.table_exists(table): - raise NotFound("Table {} does not exist".format(table)) - # Ensure the column exists and is of type BLOB - column_types = {c.name: c.type for c in await db.table_column_details(table)} - if column not in column_types: - raise NotFound("Table {} does not have column {}".format(table, column)) - if column_types[column].upper() not in ("BLOB", ""): - raise NotFound( - "Table {} does not have column {} of type BLOB".format(table, column) - ) - # Ensure the row exists for the pk_path - pk_values = urlsafe_components(pk_path) - sql, params, _ = await _sql_params_pks(db, table, pk_values) - results = await db.execute(sql, params, truncate=True) - rows = list(results.rows) - if not rows: - raise NotFound("Record not found: {}".format(pk_values)) - - # Serve back the binary data - filename_bits = [to_css_class(table), pk_path, to_css_class(column)] - filename = "-".join(filename_bits) + ".blob" - headers = { - "X-Content-Type-Options": "nosniff", - "Content-Disposition": 'attachment; filename="{}"'.format(filename), - } - return Response( - body=rows[0][column] or b"", - status=200, - headers=headers, - content_type="application/binary", - ) diff --git a/docs/pages.rst b/docs/pages.rst index 3ad58565..db970ead 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -77,14 +77,3 @@ Note that this URL includes the encoded primary key of the record. Here's that same page as JSON: `../people/uk.org.publicwhip%2Fperson%2F10001.json `_ - -.. _BlobView: - -Blob -==== - -SQLite databases can contain binary data, stored in a ``BLOB`` column. Datasette makes the content of these columns available to download directly, at URLs that look like the following:: - - /database-name/table-name/-/blob/row-identifier/column-name.blob - -Binary content is also made available as a base64 encoded string in the ``.json`` representation of the row. diff --git a/tests/test_csv.py b/tests/test_csv.py index 86e402b5..863659f7 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,3 +1,5 @@ +import textwrap +import pytest from .fixtures import ( # noqa app_client, app_client_csv_max_mb_one, @@ -78,6 +80,22 @@ def test_table_csv_with_nullable_labels(app_client): assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text +@pytest.mark.xfail +def test_table_csv_blob_columns(app_client): + response = app_client.get("/fixtures/binary_data.csv") + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert EXPECTED_TABLE_CSV == textwrap.dedent( + """ + rowid,data + 1,/fixtures/binary_data/-/blob/1/data.blob + 2,/fixtures/binary_data/-/blob/1/data.blob + """.strip().replace( + "\n", "\r\n" + ) + ) + + def test_custom_sql_csv(app_client): response = app_client.get( "/fixtures.csv?sql=select+content+from+simple_primary_key+limit+2" diff --git a/tests/test_html.py b/tests/test_html.py index 3c1101d2..95b5128a 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1223,7 +1223,7 @@ def test_extra_where_clauses(app_client): ] -def test_binary_data_display(app_client): +def test_binary_data_display_in_table(app_client): response = app_client.get("/fixtures/binary_data") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -1231,12 +1231,12 @@ def test_binary_data_display(app_client): [ '', '', - '', + '', ], [ '', '', - '', + '', ], [ '', @@ -1249,21 +1249,38 @@ def test_binary_data_display(app_client): ] +def test_binary_data_display_in_query(app_client): + response = app_client.get("/fixtures?sql=select+*+from+binary_data") + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + expected_tds = [ + [ + '' + ], + [ + '' + ], + [''], + ] + assert expected_tds == [ + [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") + ] + + @pytest.mark.parametrize( - "path,expected_body,expected_filename", + "path,expected_filename", [ + ("/fixtures/binary_data/1.blob?_blob_column=data", "binary_data-1-data.blob"), ( - "/fixtures/binary_data/-/blob/1/data.blob", - b"\x15\x1c\x02\xc7\xad\x05\xfe", - "binary_data-1-data.blob", + "/fixtures.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d", + "data-f30889.blob", ), - ("/fixtures/binary_data/-/blob/3/data.blob", b"", "binary_data-3-data.blob"), ], ) -def test_blob_download(app_client, path, expected_body, expected_filename): +def test_blob_download(app_client, path, expected_filename): response = app_client.get(path) assert response.status == 200 - assert response.body == expected_body + assert response.body == b"\x15\x1c\x02\xc7\xad\x05\xfe" assert response.headers["x-content-type-options"] == "nosniff" assert response.headers[ "content-disposition" @@ -1274,28 +1291,17 @@ def test_blob_download(app_client, path, expected_body, expected_filename): @pytest.mark.parametrize( "path,expected_message", [ - ("/baddb/binary_data/-/blob/1/data.blob", "Database baddb does not exist"), + ("/fixtures/binary_data/1.blob", "?_blob_column= is required"), + ("/fixtures/binary_data/1.blob?_blob_column=foo", "foo is not a valid column"), ( - "/fixtures/binary_data_bad/-/blob/1/data.blob", - "Table binary_data_bad does not exist", - ), - ( - "/fixtures/binary_data/-/blob/1/bad.blob", - "Table binary_data does not have column bad", - ), - ( - "/fixtures/facetable/-/blob/1/state.blob", - "Table facetable does not have column state of type BLOB", - ), - ( - "/fixtures/binary_data/-/blob/101/data.blob", - "Record not found: ['101']", + "/fixtures/binary_data/1.blob?_blob_column=data&_blob_hash=x", + "Link has expired - the requested binary content has changed or could not be found.", ), ], ) -def test_blob_download_not_found_messages(app_client, path, expected_message): +def test_blob_download_invalid_messages(app_client, path, expected_message): response = app_client.get(path) - assert response.status == 404 + assert response.status == 400 assert expected_message in response.text diff --git a/tests/test_permissions.py b/tests/test_permissions.py index a935a495..4d1b09b8 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -417,17 +417,6 @@ def cascade_app_client(): ("/fixtures/binary_data/1", ["table"], 200), ("/fixtures/binary_data/1", ["table", "database"], 200), ("/fixtures/binary_data/1", ["table", "database", "instance"], 200), - # ... and for binary blob - ("/fixtures/binary_data/-/blob/1/data.blob", [], 403), - ("/fixtures/binary_data/-/blob/1/data.blob", ["database"], 403), - ("/fixtures/binary_data/-/blob/1/data.blob", ["instance"], 403), - ("/fixtures/binary_data/-/blob/1/data.blob", ["table"], 200), - ("/fixtures/binary_data/-/blob/1/data.blob", ["table", "database"], 200), - ( - "/fixtures/binary_data/-/blob/1/data.blob", - ["table", "database", "instance"], - 200, - ), # Can view query even if not allowed database or instance ("/fixtures/magic_parameters", [], 403), ("/fixtures/magic_parameters", ["database"], 403), From 178b7e8749f14300363af5961e9f8964595264d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 15:47:32 -0700 Subject: [PATCH 0124/1866] .csv now links to .blob downloads Closes #1063, closes #1034 --- datasette/utils/__init__.py | 6 ++++-- datasette/views/base.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_csv.py | 28 +++++++++++++++++----------- tests/test_utils.py | 8 ++++++++ 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index f819aa82..33decbfc 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -678,10 +678,12 @@ async def resolve_table_and_format( return table_and_format, None -def path_with_format(request, format, extra_qs=None): +def path_with_format(request, format, extra_qs=None, replace_format=None): qs = extra_qs or {} path = request.path - if "." in request.path: + if replace_format and path.endswith(".{}".format(replace_format)): + path = path[: -(1 + len(replace_format))] + if "." in path: qs["_format"] = format else: path = "{}.{}".format(path, format) diff --git a/datasette/views/base.py b/datasette/views/base.py index 4432ddca..6ca78934 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,5 +1,6 @@ import asyncio import csv +import hashlib import re import time import urllib @@ -14,6 +15,7 @@ from datasette.utils import ( InvalidSql, LimitedWriter, call_with_supported_arguments, + path_from_row_pks, path_with_added_args, path_with_removed_args, path_with_format, @@ -310,6 +312,40 @@ class DataView(BaseView): first = False next = data.get("next") for row in data["rows"]: + if any(isinstance(r, bytes) for r in row): + new_row = [] + for column, cell in zip(headings, row): + if isinstance(cell, bytes): + # If this is a table page, use .urls.row_blob() + if data.get("table"): + pks = data.get("primary_keys") or [] + cell = self.ds.absolute_url( + request, + self.ds.urls.row_blob( + database, + data["table"], + path_from_row_pks(row, pks, not pks), + column, + ), + ) + else: + # Otherwise generate URL for this query + cell = self.ds.absolute_url( + request, + path_with_format( + request, + "blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256( + cell + ).hexdigest(), + }, + replace_format="csv", + ), + ) + new_row.append(cell) + row = new_row if not expanded_columns: # Simple path await writer.writerow(row) diff --git a/tests/test_csv.py b/tests/test_csv.py index 863659f7..1a701828 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,5 +1,3 @@ -import textwrap -import pytest from .fixtures import ( # noqa app_client, app_client_csv_max_mb_one, @@ -80,19 +78,27 @@ def test_table_csv_with_nullable_labels(app_client): assert EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV == response.text -@pytest.mark.xfail def test_table_csv_blob_columns(app_client): response = app_client.get("/fixtures/binary_data.csv") assert response.status == 200 assert "text/plain; charset=utf-8" == response.headers["content-type"] - assert EXPECTED_TABLE_CSV == textwrap.dedent( - """ - rowid,data - 1,/fixtures/binary_data/-/blob/1/data.blob - 2,/fixtures/binary_data/-/blob/1/data.blob - """.strip().replace( - "\n", "\r\n" - ) + assert response.text == ( + "rowid,data\r\n" + "1,http://localhost/fixtures/binary_data/1.blob?_blob_column=data\r\n" + "2,http://localhost/fixtures/binary_data/2.blob?_blob_column=data\r\n" + "3,\r\n" + ) + + +def test_custom_sql_csv_blob_columns(app_client): + response = app_client.get("/fixtures.csv?sql=select+rowid,+data+from+binary_data") + assert response.status == 200 + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert response.text == ( + "rowid,data\r\n" + '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' + '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' + "3,\r\n" ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0e2af098..bae3b685 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -386,6 +386,14 @@ def test_path_with_format(path, format, extra_qs, expected): assert expected == actual +def test_path_with_format_replace_format(): + request = Request.fake("/foo/bar.csv") + assert utils.path_with_format(request, "blob") == "/foo/bar.csv?_format=blob" + assert ( + utils.path_with_format(request, "blob", replace_format="csv") == "/foo/bar.blob" + ) + + @pytest.mark.parametrize( "bytes,expected", [ From 1a861be19e326e0c88230a711a1b6536366697d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 15:58:40 -0700 Subject: [PATCH 0125/1866] Fixed test_max_csv_mb test that I just broke, refs #1063 --- tests/test_csv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index 1a701828..3e91fb04 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -130,8 +130,10 @@ def test_csv_with_non_ascii_characters(app_client): def test_max_csv_mb(app_client_csv_max_mb_one): response = app_client_csv_max_mb_one.get( - "/fixtures.csv?sql=select+randomblob(10000)+" - "from+compound_three_primary_keys&_stream=1&_size=max" + ( + "/fixtures.csv?sql=select+'{}'+" + "from+compound_three_primary_keys&_stream=1&_size=max" + ).format("abcdefg" * 10000) ) # It's a 200 because we started streaming before we knew the error assert response.status == 200 From 18a64fbb29271ce607937110bbdb55488c43f4e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 20:45:15 -0700 Subject: [PATCH 0126/1866] Navigation menu plus menu_links() hook Closes #1064, refs #690. --- datasette/app.py | 12 +++++++++ datasette/default_menu_links.py | 40 +++++++++++++++++++++++++++ datasette/hookspecs.py | 5 ++++ datasette/plugins.py | 1 + datasette/static/app.css | 31 ++++++++++++++++++--- datasette/templates/base.html | 48 ++++++++++++++++++++++++++++----- docs/plugin_hooks.rst | 32 ++++++++++++++++++++++ tests/fixtures.py | 2 ++ tests/plugins/my_plugin.py | 6 +++++ tests/plugins/my_plugin_2.py | 9 +++++++ tests/test_auth.py | 3 +-- tests/test_plugins.py | 17 ++++++++++++ 12 files changed, 193 insertions(+), 13 deletions(-) create mode 100644 datasette/default_menu_links.py diff --git a/datasette/app.py b/datasette/app.py index 3016043a..fb5c34a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -750,11 +750,22 @@ class Datasette: ) extra_template_vars.update(extra_vars) + async def menu_links(): + links = [] + for hook in pm.hook.menu_links( + datasette=self, actor=request.actor if request else None + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + template_context = { **context, **{ "urls": self.urls, "actor": request.actor if request else None, + "menu_links": menu_links, "display_actor": display_actor, "show_logout": request is not None and "ds_actor" in request.cookies, "app_css_hash": self.app_css_hash(), @@ -1161,6 +1172,7 @@ class DatasetteRouter: info, urls=self.ds.urls, app_css_hash=self.ds.app_css_hash(), + menu_links=lambda: [], ) ), status=status, diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py new file mode 100644 index 00000000..11374fb5 --- /dev/null +++ b/datasette/default_menu_links.py @@ -0,0 +1,40 @@ +from datasette import hookimpl + + +@hookimpl +def menu_links(datasette, actor): + if actor and actor.get("id") == "root": + return [ + {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, + { + "href": datasette.urls.path("/-/plugins"), + "label": "Installed plugins", + }, + { + "href": datasette.urls.path("/-/versions"), + "label": "Version info", + }, + { + "href": datasette.urls.path("/-/metadata"), + "label": "Metadata", + }, + { + "href": datasette.urls.path("/-/config"), + "label": "Config", + }, + { + "href": datasette.urls.path("/-/permissions"), + "label": "Debug permissions", + }, + { + "href": datasette.urls.path("/-/messages"), + "label": "Debug messages", + }, + { + "href": datasette.urls.path("/-/allow-debug"), + "label": "Debug allow rules", + }, + {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, + {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, + {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, + ] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f7e90e4e..7bad262a 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -97,3 +97,8 @@ def register_magic_parameters(datasette): @hookspec def forbidden(datasette, request, message): "Custom response for a 403 forbidden error" + + +@hookspec +def menu_links(datasette, actor): + "Links for the navigation menu" diff --git a/datasette/plugins.py b/datasette/plugins.py index 1c2f392f..50791988 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -13,6 +13,7 @@ DEFAULT_PLUGINS = ( "datasette.default_permissions", "datasette.default_magic_parameters", "datasette.blob_renderer", + "datasette.default_menu_links", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/static/app.css b/datasette/static/app.css index 8b462b35..2fd5371b 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -261,13 +261,13 @@ footer p { header .crumbs { float: left; } -header .logout { +header .actor { float: right; text-align: right; padding-left: 1rem; -} -header .logout form { - display: inline; + padding-right: 1rem; + position: relative; + top: -3px; } footer a:link, @@ -312,6 +312,29 @@ footer { margin-top: 1rem; } +/* Navigation menu */ +details.nav-menu > summary { + list-style: none; + display: inline; + float: right; + position: relative; +} +details.nav-menu > summary::-webkit-details-marker { + display: none; +} +details .nav-menu-inner { + position: absolute; + top: 2rem; + right: 10px; + width: 180px; + background-color: #276890; + padding: 1rem; + z-index: 1000; +} +.nav-menu-inner a { + display: block; +} + /* Components ============================================================== */ diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 03de2115..ec1fd00e 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -13,15 +13,33 @@ {% block extra_head %}{% endblock %} -
    @@ -41,6 +59,22 @@
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    + {% for body_script in body_scripts %} {% endfor %} diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b2c62ccd..82bc56a9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -989,3 +989,35 @@ The function can alternatively return an awaitable function if it needs to make return Response.html(await datasette.render_template("forbidden.html")) return inner + +.. _plugin_hook_menu_links: + +menu_links(datasette, actor) +---------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``request`` - object + The current HTTP :ref:`internals_request`. + +This hook provides items to be included in the menu displayed by Datasette's top right menu icon. + +The hook should return a list of ``{"href": "...", "label": "..."}`` menu items. These will be added to the menu. + +It can alternatively return an ``async def`` awaitable function which returns a list of menu items. + +This example adds a new menu item but only if the signed in user is ``"root"``: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def menu_links(datasette, actor): + if actor and actor.get("id") == "root": + return [ + {"href": datasette.urls.path("/-/edit-schema"), "label": "Edit schema"}, + ] + +Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`config_base_url` setting into account. diff --git a/tests/fixtures.py b/tests/fixtures.py index 31638fc8..69853b7d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -43,6 +43,7 @@ EXPECTED_PLUGINS = [ "extra_js_urls", "extra_template_vars", "forbidden", + "menu_links", "permission_allowed", "prepare_connection", "prepare_jinja2_environment", @@ -64,6 +65,7 @@ EXPECTED_PLUGINS = [ "canned_queries", "extra_js_urls", "extra_template_vars", + "menu_links", "permission_allowed", "render_cell", "startup", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 0dd0ad26..7f8a4871 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -290,3 +290,9 @@ def forbidden(datasette, request, message): datasette._last_forbidden_message = message if request.path == "/data2": return Response.redirect("/login?message=" + message) + + +@hookimpl +def menu_links(datasette, actor): + if actor: + return [{"href": datasette.urls.instance(), "label": "Hello"}] diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index ae0f338a..981b24cc 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -146,3 +146,12 @@ def canned_queries(datasette, database): } return inner + + +@hookimpl(trylast=True) +def menu_links(datasette, actor): + async def inner(): + if actor: + return [{"href": datasette.urls.instance(), "label": "Hello 2"}] + + return inner diff --git a/tests/test_auth.py b/tests/test_auth.py index f244f268..34138aa6 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -101,7 +101,7 @@ def test_logout_button_in_navigation(app_client, path): ) anon_response = app_client.get(path) for fragment in ( - "test ·", + "test", '
    ', ): assert fragment in response.text @@ -112,5 +112,4 @@ def test_logout_button_in_navigation(app_client, path): def test_no_logout_button_in_navigation_if_no_ds_actor_cookie(app_client, path): response = app_client.get(path + "?_bot=1") assert "bot" in response.text - assert "bot ·" not in response.text assert '' not in response.text diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 08ed2e6b..191d943d 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -765,3 +765,20 @@ def test_hook_forbidden(restore_working_directory): assert 302 == response2.status assert "/login?message=view-database" == response2.headers["Location"] assert "view-database" == client.ds._last_forbidden_message + + +def test_hook_menu_links(app_client): + def get_menu_links(html): + soup = Soup(html, "html.parser") + return [ + {"label": a.text, "href": a["href"]} for a in soup.find("nav").select("a") + ] + + response = app_client.get("/") + assert get_menu_links(response.text) == [] + + response_2 = app_client.get("/?_bot=1") + assert get_menu_links(response_2.text) == [ + {"label": "Hello", "href": "/"}, + {"label": "Hello 2", "href": "/"}, + ] From 561c1d2d36a89675764e4410a2a127323402eaa3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 20:51:37 -0700 Subject: [PATCH 0127/1866] Show logout link if they are logged in AND have ds_actor cookie Otherwise an expired cookie will still cause the logout link to show. --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index fb5c34a4..efe5a812 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -767,7 +767,7 @@ class Datasette: "actor": request.actor if request else None, "menu_links": menu_links, "display_actor": display_actor, - "show_logout": request is not None and "ds_actor" in request.cookies, + "show_logout": request is not None and "ds_actor" in request.cookies and request.actor, "app_css_hash": self.app_css_hash(), "zip": zip, "body_scripts": body_scripts, From 8a4639bc43a016a1f8fae6a07d5b5f7abe0074e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 22:14:33 -0700 Subject: [PATCH 0128/1866] Applied Black --- datasette/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index efe5a812..8cff6577 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -767,7 +767,9 @@ class Datasette: "actor": request.actor if request else None, "menu_links": menu_links, "display_actor": display_actor, - "show_logout": request is not None and "ds_actor" in request.cookies and request.actor, + "show_logout": request is not None + and "ds_actor" in request.cookies + and request.actor, "app_css_hash": self.app_css_hash(), "zip": zip, "body_scripts": body_scripts, From 2f7731e9e5ff9b324beb5039fbe2be55d704a184 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 22:16:41 -0700 Subject: [PATCH 0129/1866] table_actions() plugin hook plus menu, closes #1066 Refs #690 --- datasette/hookspecs.py | 5 ++++ datasette/static/app.css | 34 ++++++++++++++++++++++++++- datasette/templates/base.html | 16 ++++++------- datasette/templates/table.html | 25 ++++++++++++++++++-- datasette/views/table.py | 16 +++++++++++++ docs/plugin_hooks.rst | 42 +++++++++++++++++++++++++++++++--- tests/fixtures.py | 2 ++ tests/plugins/my_plugin.py | 12 ++++++++++ tests/plugins/my_plugin_2.py | 9 ++++++++ tests/test_plugins.py | 19 +++++++++++++++ 10 files changed, 166 insertions(+), 14 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 7bad262a..78070e67 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -102,3 +102,8 @@ def forbidden(datasette, request, message): @hookspec def menu_links(datasette, actor): "Links for the navigation menu" + + +@hookspec +def table_actions(datasette, actor, database, table): + "Links for the table actions menu" diff --git a/datasette/static/app.css b/datasette/static/app.css index 2fd5371b..95457766 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -118,7 +118,7 @@ h6, .header3, .header4, .header5, -.header6 { +.header6 { font-weight: 700; font-size: 1rem; margin: 0; @@ -162,6 +162,29 @@ h6, text-decoration: underline; } +.page-header { + padding-left: 10px; + border-left: 10px solid #666; + margin-bottom: 0.75rem; + margin-top: 1rem; +} +.page-header h1 { + display: inline; + margin: 0; + font-size: 2rem; + padding-right: 0.2em; +} +.page-header details { + display: inline; +} +.page-header details > summary { + list-style: none; + display: inline; +} +.page-header details > summary::-webkit-details-marker { + display: none; +} + div, section, article, @@ -335,6 +358,15 @@ details .nav-menu-inner { display: block; } +/* Table actions menu */ +.table-menu-links { + position: relative; +} +.table-menu-links .dropdown-menu { + position: absolute; + top: 2rem; + right: 0; +} /* Components ============================================================== */ diff --git a/datasette/templates/base.html b/datasette/templates/base.html index ec1fd00e..d860df37 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -60,19 +60,19 @@
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    {% for body_script in body_scripts %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 6c27beee..13f6a832 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -25,8 +25,29 @@ {% endblock %} {% block content %} - -

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

    +
    +

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

    + {% set links = table_actions() %}{% if links %} +
    + + Table actions + + + +
    + {% if links %} + + {% endif %} +
    +
    {% endif %} +
    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/table.py b/datasette/views/table.py index 079e0b0a..65fe7f8b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -7,6 +7,7 @@ import jinja2 from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( + await_me_maybe, CustomRow, MultiParams, append_querystring, @@ -840,7 +841,21 @@ class TableView(RowTableShared): elif use_rowid: sort = "rowid" + async def table_actions(): + links = [] + for hook in pm.hook.table_actions( + datasette=self.ds, + table=table, + database=database, + actor=request.actor, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + return { + "table_actions": table_actions, "supports_search": bool(fts_table), "search": search or "", "use_rowid": use_rowid, @@ -959,6 +974,7 @@ class RowView(RowTableShared): ) for column in display_columns: column["sortable"] = False + return { "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 82bc56a9..1c28c72e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -998,10 +998,10 @@ menu_links(datasette, actor) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - object - The current HTTP :ref:`internals_request`. +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. -This hook provides items to be included in the menu displayed by Datasette's top right menu icon. +This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. The hook should return a list of ``{"href": "...", "label": "..."}`` menu items. These will be added to the menu. @@ -1021,3 +1021,39 @@ This example adds a new menu item but only if the signed in user is ``"root"``: ] Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`config_base_url` setting into account. + + +.. _plugin_hook_table_actions: + +table_actions(datasette, actor, database, table) +------------------------------------------------ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. + +It can alternatively return an ``async def`` awaitable function which returns a list of menu items. + +This example adds a new table action if the signed in user is ``"root"``: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def table_actions(datasette, actor): + if actor and actor.get("id") == "root": + return [{ + "href": datasette.urls.path("/-/edit-schema/{}/{}".format(database, table)), + "label": "Edit schema for this table", + }] diff --git a/tests/fixtures.py b/tests/fixtures.py index 69853b7d..2f8383ef 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -52,6 +52,7 @@ EXPECTED_PLUGINS = [ "register_routes", "render_cell", "startup", + "table_actions", ], }, { @@ -69,6 +70,7 @@ EXPECTED_PLUGINS = [ "permission_allowed", "render_cell", "startup", + "table_actions", ], }, { diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 7f8a4871..8fc6a1b4 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -296,3 +296,15 @@ def forbidden(datasette, request, message): def menu_links(datasette, actor): if actor: return [{"href": datasette.urls.instance(), "label": "Hello"}] + + +@hookimpl +def table_actions(datasette, database, table, actor): + if actor: + return [ + { + "href": datasette.urls.instance(), + "label": "Database: {}".format(database), + }, + {"href": datasette.urls.instance(), "label": "Table: {}".format(table)}, + ] diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 981b24cc..7d8095ed 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -155,3 +155,12 @@ def menu_links(datasette, actor): return [{"href": datasette.urls.instance(), "label": "Hello 2"}] return inner + + +@hookimpl +def table_actions(datasette, database, table, actor): + async def inner(): + if actor: + return [{"href": datasette.urls.instance(), "label": "From async"}] + + return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 191d943d..be36a517 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -782,3 +782,22 @@ def test_hook_menu_links(app_client): {"label": "Hello", "href": "/"}, {"label": "Hello 2", "href": "/"}, ] + + +def test_hook_table_actions(app_client): + def get_table_actions_links(html): + soup = Soup(html, "html.parser") + details = soup.find("details", {"class": "table-menu-links"}) + if details is None: + return [] + return [{"label": a.text, "href": a["href"]} for a in details.select("a")] + + response = app_client.get("/fixtures/facetable") + assert get_table_actions_links(response.text) == [] + + response_2 = app_client.get("/fixtures/facetable?_bot=1") + assert get_table_actions_links(response_2.text) == [ + {"label": "From async", "href": "/"}, + {"label": "Database: fixtures", "href": "/"}, + {"label": "Table: facetable", "href": "/"}, + ] From 0e1e89c6ba3d0fbdb0823272952cf356f3016def Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 22:34:52 -0700 Subject: [PATCH 0130/1866] Release 0.51a1 Refs #1056, #1039, #998, #1045, #1033, #1036, #1034, #976, #1057, #1058, #1053, #1064, #1066 --- datasette/version.py | 2 +- docs/changelog.rst | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b57d7a12..9a89c8e6 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.51.a0" +__version__ = "0.51a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 17309155..893a0ee5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,25 @@ Changelog ========= +.. _v0_51_a1: + +0.51a1 (2020-10-29) +------------------- + +- New colour scheme and improved visual design, courtesy of Natalie Downe. (`#1056 `__) +- scale-in animation for column action menu. (`#1039 `__) +- Wide tables now scroll horizontally. (`#998 `__) +- Option to pass a list of templates to ``.render_template()`` is now documented. (`#1045 `__) +- New ``datasette.urls.static_plugins()`` method. (`#1033 `__) +- ``BLOB`` column values can now be downloaded directly from the Datasette UI. (`#1036 `__) +- ``.csv`` exports now link to direct ``BLOB`` downloads. (`#1034 `__) +- ``datasette -o`` option now opens the most relevant page. (`#976 `__) +- ``datasette --cors`` option now enables access to ``/database.db`` downloads. (`#1057 `__) +- Database file downloads now implement cascading permissions, so you can download a database if you have ``view-database-download`` permission even if you do not have permission to access the Datasette instance. (`#1058 `__) +- New documentation on :ref:`writing_plugins_designing_urls`. (`#1053 `__) +- New navigation menu plus a :ref:`plugin_hook_menu_links` plugin hook to customize it. (`#1064 `__) +- :ref:`plugin_hook_table_actions` plugin hook for the new table actions menu. (`#1066 `__) + .. _v0_51_a0: 0.51a0 (2020-10-19) From 9f0987cb57a82a7d2fe0c679fc909e5b39593ee4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Oct 2020 22:55:10 -0700 Subject: [PATCH 0131/1866] cursor: pointer; on the new menu icons Refs #1064, #1066 --- datasette/static/app.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 95457766..a1eb2099 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -180,6 +180,7 @@ h6, .page-header details > summary { list-style: none; display: inline; + cursor: pointer; } .page-header details > summary::-webkit-details-marker { display: none; @@ -341,6 +342,7 @@ details.nav-menu > summary { display: inline; float: right; position: relative; + cursor: pointer; } details.nav-menu > summary::-webkit-details-marker { display: none; From 222f79bb4c6e2aa5426cc5ff25f1b2461e18a300 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 08:41:57 -0700 Subject: [PATCH 0132/1866] debug-menu permission, closes #1068 Also added tests for navigation menu logic. --- datasette/default_menu_links.py | 7 +++++- datasette/default_permissions.py | 2 +- datasette/views/special.py | 3 ++- docs/authentication.rst | 9 ++++++++ tests/test_html.py | 38 ++++++++++++++++++++++++++++++++ tests/test_permissions.py | 3 ++- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py index 11374fb5..0b135410 100644 --- a/datasette/default_menu_links.py +++ b/datasette/default_menu_links.py @@ -3,7 +3,10 @@ from datasette import hookimpl @hookimpl def menu_links(datasette, actor): - if actor and actor.get("id") == "root": + async def inner(): + if not await datasette.permission_allowed(actor, "debug-menu"): + return [] + return [ {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, { @@ -38,3 +41,5 @@ def menu_links(datasette, actor): {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, ] + + return inner diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index ddd45940..9f1d9c62 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -5,7 +5,7 @@ from datasette.utils import actor_matches_allow @hookimpl(tryfirst=True) def permission_allowed(datasette, actor, action, resource): async def inner(): - if action == "permissions-debug": + if action in ("permissions-debug", "debug-menu"): if actor and actor.get("id") == "root": return True elif action == "view-instance": diff --git a/datasette/views/special.py b/datasette/views/special.py index a9fc59b7..397dbc8c 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -96,7 +96,8 @@ class PermissionsDebugView(BaseView): return await self.render( ["permissions_debug.html"], request, - {"permission_checks": reversed(self.ds._permission_checks)}, + # list() avoids error if check is performed during template render: + {"permission_checks": list(reversed(self.ds._permission_checks))}, ) diff --git a/docs/authentication.rst b/docs/authentication.rst index f6c5d801..62ed7e8b 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -522,3 +522,12 @@ permissions-debug Actor is allowed to view the ``/-/permissions`` debug page. Default *deny*. + +.. _permissions_debug_menu: + +debug-menu +---------- + +Controls if the various debug pages are displayed in the navigation menu. + +Default *deny*. diff --git a/tests/test_html.py b/tests/test_html.py index 95b5128a..fed643a9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1507,3 +1507,41 @@ def test_edit_sql_link_not_shown_if_user_lacks_permission(permission_allowed): assert "Edit SQL" in response.text else: assert "Edit SQL" not in response.text + + +@pytest.mark.parametrize( + "actor_id,should_have_links,should_not_have_links", + [ + (None, None, None), + ("test", None, ["/-/permissions"]), + ("root", ["/-/permissions", "/-/allow-debug", "/-/metadata"], None), + ], +) +def test_navigation_menu_links( + app_client, actor_id, should_have_links, should_not_have_links +): + cookies = {} + if actor_id: + cookies = {"ds_actor": app_client.actor_cookie({"id": actor_id})} + html = app_client.get("/", cookies=cookies).text + soup = Soup(html, "html.parser") + details = soup.find("nav").find("details") + if not actor_id: + # Should not show a menu + assert details is None + return + # They are logged in: should show a menu + assert details is not None + # And a rogout form + assert details.find("form") is not None + if should_have_links: + for link in should_have_links: + assert ( + details.find("a", {"href": link}) is not None + ), "{} expected but missing from nav menu".format(link) + + if should_not_have_links: + for link in should_not_have_links: + assert ( + details.find("a", {"href": link}) is None + ), "{} found but should not have been in nav menu".format(link) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4d1b09b8..60883eef 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -310,10 +310,11 @@ def test_permissions_checked(app_client, path, permissions): def test_permissions_debug(app_client): app_client.ds._permission_checks.clear() - assert 403 == app_client.get("/-/permissions").status + assert app_client.get("/-/permissions").status == 403 # With the cookie it should work cookie = app_client.actor_cookie({"id": "root"}) response = app_client.get("/-/permissions", cookies={"ds_actor": cookie}) + assert response.status == 200 # Should show one failure and one success soup = Soup(response.body, "html.parser") check_divs = soup.findAll("div", {"class": "check"}) From fcf43589eb6a1f1d0432772a639fd35711c48e0c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 08:53:44 -0700 Subject: [PATCH 0133/1866] Link to homepage in nav on show-json page --- datasette/templates/show_json.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datasette/templates/show_json.html b/datasette/templates/show_json.html index b9e49eb2..fd88756f 100644 --- a/datasette/templates/show_json.html +++ b/datasette/templates/show_json.html @@ -4,6 +4,13 @@ {% block body_class %}show-json{% endblock %} +{% block nav %} +

    + home +

    + {{ super() }} +{% endblock %} + {% block content %}

    {{ filename }}

    From 81dea4b07ab2b6f4eaaf248307d2b588472054a1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 10:47:18 -0700 Subject: [PATCH 0134/1866] load_template() plugin hook Closes #1042 --- datasette/app.py | 34 ++++++++++++++++++++++++++++++++-- datasette/hookspecs.py | 5 +++++ datasette/templates/base.html | 6 +++++- datasette/views/base.py | 10 +--------- docs/plugin_hooks.rst | 18 ++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 6 ++++++ tests/test_plugins.py | 5 +++++ 8 files changed, 73 insertions(+), 12 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8cff6577..4b28e715 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -21,7 +21,7 @@ from pathlib import Path from markupsafe import Markup from itsdangerous import URLSafeSerializer import jinja2 -from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape +from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound import uvicorn @@ -713,12 +713,41 @@ class Datasette: self, templates, context=None, request=None, view_name=None ): context = context or {} + templates_considered = [] if isinstance(templates, Template): template = templates else: if isinstance(templates, str): templates = [templates] - template = self.jinja_env.select_template(templates) + + # Give plugins first chance at loading the template + break_outer = False + plugin_template_source = None + plugin_template_name = None + template_name = None + for template_name in templates: + if break_outer: + break + plugin_template_source = pm.hook.load_template( + template=template_name, + request=request, + datasette=self, + ) + plugin_template_source = await await_me_maybe(plugin_template_source) + if plugin_template_source: + break_outer = True + plugin_template_name = template_name + break + if plugin_template_source is not None: + template = self.jinja_env.from_string(plugin_template_source) + else: + template = self.jinja_env.select_template(templates) + for template_name in templates: + from_plugin = template_name == plugin_template_name + used = from_plugin or template_name == template.name + templates_considered.append( + {"name": template_name, "used": used, "from_plugin": from_plugin} + ) body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( @@ -783,6 +812,7 @@ class Datasette: ), "base_url": self.config("base_url"), "csrftoken": request.scope["csrftoken"] if request else lambda: "", + "templates_considered": templates_considered, }, **extra_template_vars, } diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 78070e67..ca84b355 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -49,6 +49,11 @@ def extra_template_vars( "Extra template variables to be made available to the template - can return dict or callable or awaitable" +@hookspec(firstresult=True) +def load_template(template, request, datasette): + "Load the specified template, returning the template code as a string" + + @hookspec def publish_subcommand(publish): "Subcommands for 'datasette publish'" diff --git a/datasette/templates/base.html b/datasette/templates/base.html index d860df37..e29c2ea5 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -79,6 +79,10 @@ document.body.addEventListener('click', (ev) => { {% endfor %} -{% if select_templates %}{% endif %} +{% if templates_considered %} + +{% endif %} diff --git a/datasette/views/base.py b/datasette/views/base.py index 6ca78934..ed2631c5 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -8,7 +8,6 @@ import urllib import pint from datasette import __version__ -from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, @@ -119,22 +118,15 @@ class BaseView: async def render(self, templates, request, context=None): context = context or {} - template = self.ds.jinja_env.select_template(templates) template_context = { **context, **{ "database_color": self.database_color, - "select_templates": [ - "{}{}".format( - "*" if template_name == template.name else "", template_name - ) - for template_name in templates - ], }, } return Response.html( await self.ds.render_template( - template, template_context, request=request, view_name=self.name + templates, template_context, request=request, view_name=self.name ) ) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1c28c72e..3c57b6a8 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -271,6 +271,24 @@ You can also return an awaitable function that returns a string. Example: `datasette-cluster-map `_ +.. _plugin_hook_load_template: + +load_template(template, request, datasette) +------------------------------------------- + +``template`` - string + The template that is being rendered, e.g. ``database.html`` + +``request`` - object or None + The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` + +Load the source code for a template from a custom location. Hooks should return a string, or ``None`` if the template is not found. + +Datasette will fall back to serving templates from files on disk if the requested template cannot be loaded by any plugins. + .. _plugin_hook_publish_subcommand: publish_subcommand(publish) diff --git a/tests/fixtures.py b/tests/fixtures.py index 2f8383ef..9f3052b7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -43,6 +43,7 @@ EXPECTED_PLUGINS = [ "extra_js_urls", "extra_template_vars", "forbidden", + "load_template", "menu_links", "permission_allowed", "prepare_connection", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 8fc6a1b4..9dbb3f40 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -308,3 +308,9 @@ def table_actions(datasette, database, table, actor): }, {"href": datasette.urls.instance(), "label": "Table: {}".format(table)}, ] + + +@hookimpl +def load_template(template, request): + if template == "show_json.html" and request.args.get("_special"): + return "

    Special show_json: {{ filename }}

    " diff --git a/tests/test_plugins.py b/tests/test_plugins.py index be36a517..f8888798 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -801,3 +801,8 @@ def test_hook_table_actions(app_client): {"label": "Database: fixtures", "href": "/"}, {"label": "Table: facetable", "href": "/"}, ] + + +def test_hook_load_template(app_client): + response = app_client.get("/-/databases?_special=1") + assert response.text == "

    Special show_json: databases.json

    " From a7d9e24ece665eef7c6dfc5f32855c98bd45d335 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 10:52:45 -0700 Subject: [PATCH 0135/1866] Update release process with explicit version, refs #1054 --- docs/contributing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 39d4c3a2..375f6b89 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -159,12 +159,12 @@ We increment ``patch`` for bugfix releass. :ref:`contributing_alpha_beta` may have an additional ``a0`` or ``b0`` prefix - the integer component will be incremented with each subsequent alpha or beta. -To release a new version, first create a commit that updates :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: +To release a new version, first create a commit that updates the version number in ``datasette/version.py`` and the :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: # Update changelog - git commit -m "Release notes for 0.43 - - Refs #581, #770, #729, #706, #751, #706, #744, #771, #773" -a + git commit -m " Release 0.51a1 + + Refs #1056, #1039, #998, #1045, #1033, #1036, #1034, #976, #1057, #1058, #1053, #1064, #1066" -a git push Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. From 0cb29498c796267c5e4a5545ede8058b7ca03a94 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 10:54:47 -0700 Subject: [PATCH 0136/1866] Fixed bug with python tests/fixtures.py https://github.com/simonw/datasette/runs/1333357885?check_suite_focus=true --- datasette/views/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/views/base.py b/datasette/views/base.py index ed2631c5..813ee452 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -8,6 +8,7 @@ import urllib import pint from datasette import __version__ +from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, From 59ab24af6bd9b517b53162fbffac1d0116100e0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 10:56:02 -0700 Subject: [PATCH 0137/1866] Release 0.51a2 Refs #1068, #1042, #1054 --- datasette/version.py | 2 +- docs/changelog.rst | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 9a89c8e6..2f4bc37e 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.51a1" +__version__ = "0.51a2" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 893a0ee5..262400c8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _v0_51_a2: + +0.51a2 (2020-10-30) +------------------- + +- New :ref:`plugin_hook_load_template` plugin hook. (`#1042 `__) +- New :ref:`permissions_debug_menu` permission. (`#1068 `__) + .. _v0_51_a1: 0.51a1 (2020-10-29) From 393f1b49d70e9f58bc193c6a28afff4ec9459a2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 13:12:01 -0700 Subject: [PATCH 0138/1866] Updated nav in pattern portfolio --- datasette/templates/patterns.html | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index ac9e2e46..62ef1322 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -38,11 +38,8 @@ fixtures / attraction_characteristic

    -
    - testuser · - - - +
    + testuser
    From a2a709072059c6b3da365df9a332ca744c2079e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 30 Oct 2020 13:12:57 -0700 Subject: [PATCH 0139/1866] Display messages in right place, closes #1071 --- datasette/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/base.html b/datasette/templates/base.html index e29c2ea5..7e9c6c05 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -44,7 +44,6 @@ {% endif %} {% endblock %} -
    {% block messages %} {% if show_messages %} {% for message, message_type in show_messages() %} @@ -53,6 +52,7 @@ {% endif %} {% endblock %} +
    {% block content %} {% endblock %}
    From f0a740ac21cba11ded8717f49d664f9549cd2f83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 09:21:22 -0700 Subject: [PATCH 0140/1866] Remove load_plugin hook - closes #1073 Refs #1042 This reverts commit 81dea4b07ab2b6f4eaaf248307d2b588472054a1. --- datasette/app.py | 34 ++-------------------------------- datasette/hookspecs.py | 5 ----- datasette/templates/base.html | 6 +----- datasette/views/base.py | 9 ++++++++- docs/plugin_hooks.rst | 18 ------------------ tests/fixtures.py | 1 - tests/plugins/my_plugin.py | 6 ------ tests/test_plugins.py | 5 ----- 8 files changed, 11 insertions(+), 73 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4b28e715..8cff6577 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -21,7 +21,7 @@ from pathlib import Path from markupsafe import Markup from itsdangerous import URLSafeSerializer import jinja2 -from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader +from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound import uvicorn @@ -713,41 +713,12 @@ class Datasette: self, templates, context=None, request=None, view_name=None ): context = context or {} - templates_considered = [] if isinstance(templates, Template): template = templates else: if isinstance(templates, str): templates = [templates] - - # Give plugins first chance at loading the template - break_outer = False - plugin_template_source = None - plugin_template_name = None - template_name = None - for template_name in templates: - if break_outer: - break - plugin_template_source = pm.hook.load_template( - template=template_name, - request=request, - datasette=self, - ) - plugin_template_source = await await_me_maybe(plugin_template_source) - if plugin_template_source: - break_outer = True - plugin_template_name = template_name - break - if plugin_template_source is not None: - template = self.jinja_env.from_string(plugin_template_source) - else: - template = self.jinja_env.select_template(templates) - for template_name in templates: - from_plugin = template_name == plugin_template_name - used = from_plugin or template_name == template.name - templates_considered.append( - {"name": template_name, "used": used, "from_plugin": from_plugin} - ) + template = self.jinja_env.select_template(templates) body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( @@ -812,7 +783,6 @@ class Datasette: ), "base_url": self.config("base_url"), "csrftoken": request.scope["csrftoken"] if request else lambda: "", - "templates_considered": templates_considered, }, **extra_template_vars, } diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index ca84b355..78070e67 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -49,11 +49,6 @@ def extra_template_vars( "Extra template variables to be made available to the template - can return dict or callable or awaitable" -@hookspec(firstresult=True) -def load_template(template, request, datasette): - "Load the specified template, returning the template code as a string" - - @hookspec def publish_subcommand(publish): "Subcommands for 'datasette publish'" diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 7e9c6c05..611ba9f6 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -79,10 +79,6 @@ document.body.addEventListener('click', (ev) => { {% endfor %} -{% if templates_considered %} - -{% endif %} +{% if select_templates %}{% endif %} diff --git a/datasette/views/base.py b/datasette/views/base.py index 813ee452..6ca78934 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -119,15 +119,22 @@ class BaseView: async def render(self, templates, request, context=None): context = context or {} + template = self.ds.jinja_env.select_template(templates) template_context = { **context, **{ "database_color": self.database_color, + "select_templates": [ + "{}{}".format( + "*" if template_name == template.name else "", template_name + ) + for template_name in templates + ], }, } return Response.html( await self.ds.render_template( - templates, template_context, request=request, view_name=self.name + template, template_context, request=request, view_name=self.name ) ) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 3c57b6a8..1c28c72e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -271,24 +271,6 @@ You can also return an awaitable function that returns a string. Example: `datasette-cluster-map `_ -.. _plugin_hook_load_template: - -load_template(template, request, datasette) -------------------------------------------- - -``template`` - string - The template that is being rendered, e.g. ``database.html`` - -``request`` - object or None - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` - -Load the source code for a template from a custom location. Hooks should return a string, or ``None`` if the template is not found. - -Datasette will fall back to serving templates from files on disk if the requested template cannot be loaded by any plugins. - .. _plugin_hook_publish_subcommand: publish_subcommand(publish) diff --git a/tests/fixtures.py b/tests/fixtures.py index 9f3052b7..2f8383ef 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -43,7 +43,6 @@ EXPECTED_PLUGINS = [ "extra_js_urls", "extra_template_vars", "forbidden", - "load_template", "menu_links", "permission_allowed", "prepare_connection", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 9dbb3f40..8fc6a1b4 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -308,9 +308,3 @@ def table_actions(datasette, database, table, actor): }, {"href": datasette.urls.instance(), "label": "Table: {}".format(table)}, ] - - -@hookimpl -def load_template(template, request): - if template == "show_json.html" and request.args.get("_special"): - return "

    Special show_json: {{ filename }}

    " diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f8888798..be36a517 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -801,8 +801,3 @@ def test_hook_table_actions(app_client): {"label": "Database: fixtures", "href": "/"}, {"label": "Table: facetable", "href": "/"}, ] - - -def test_hook_load_template(app_client): - response = app_client.get("/-/databases?_special=1") - assert response.text == "

    Special show_json: databases.json

    " From d6db47f5c19f77e735279762d99720dc644bff48 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 10:25:32 -0700 Subject: [PATCH 0141/1866] Deploy demo plugins to latest.datasette.io, refs #1074 --- .github/workflows/deploy-latest.yml | 3 ++- tests/fixtures.py | 2 +- tests/plugins/my_plugin.py | 23 ++++++++++++++++++++++- tests/test_html.py | 8 +++++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 8445f1d8..73b97a19 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -31,7 +31,7 @@ jobs: - name: Run tests run: pytest - name: Build fixtures.db - run: python tests/fixtures.py fixtures.db fixtures.json + run: python tests/fixtures.py fixtures.db fixtures.json plugins - name: Build docs.db run: |- cd docs @@ -50,6 +50,7 @@ jobs: gcloud config set project datasette-222320 datasette publish cloudrun fixtures.db \ -m fixtures.json \ + --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ --extra-options="--config template_debug:1" \ diff --git a/tests/fixtures.py b/tests/fixtures.py index 2f8383ef..5cbfc72f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -267,7 +267,7 @@ def generate_sortable_rows(num): METADATA = { "title": "Datasette Fixtures", - "description": "An example SQLite database demonstrating Datasette", + "description_html": 'An example SQLite database demonstrating Datasette. Sign in as root user', "license": "Apache License 2.0", "license_url": "https://github.com/simonw/datasette/blob/master/LICENSE", "source": "tests/fixtures.py", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 8fc6a1b4..b487cdf0 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -177,7 +177,7 @@ def actor_from_request(datasette, request): def asgi_wrapper(): def wrap(app): async def maybe_set_actor_in_scope(scope, recieve, send): - if b"_actor_in_scope" in scope["query_string"]: + if b"_actor_in_scope" in scope.get("query_string", b""): scope = dict(scope, actor={"id": "from-scope"}) print(scope) await app(scope, recieve, send) @@ -237,12 +237,33 @@ def register_routes(): await datasette.render_template("render_message.html", request=request) ) + def login_as_root(datasette, request): + # Mainly for the latest.datasette.io demo + if request.method == "POST": + response = Response.redirect("/") + response.set_cookie( + "ds_actor", datasette.sign({"a": {"id": "root"}}, "actor") + ) + return response + return Response.html( + """ +
    +

    + +

    + + """.format( + request.path, request.scope["csrftoken"]() + ) + ) + return [ (r"/one/$", one), (r"/two/(?P.*)$", two), (r"/three/$", three), (r"/post/$", post), (r"/csrftoken-form/$", csrftoken_form), + (r"/login-as-root$", login_as_root), (r"/not-async/$", not_async), (r"/add-message/$", add_message), (r"/render-message/$", render_message), diff --git a/tests/test_html.py b/tests/test_html.py index fed643a9..7c068085 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -23,7 +23,7 @@ def test_homepage(app_client_two_attached_databases): soup = Soup(response.body, "html.parser") assert "Datasette Fixtures" == soup.find("h1").text assert ( - "An example SQLite database demonstrating Datasette" + "An example SQLite database demonstrating Datasette. Sign in as root user" == soup.select(".metadata-description")[0].text.strip() ) # Should be two attached databases @@ -949,8 +949,9 @@ def test_index_metadata(app_client): assert response.status == 200 soup = Soup(response.body, "html.parser") assert "Datasette Fixtures" == soup.find("h1").text - assert "An example SQLite database demonstrating Datasette" == inner_html( - soup.find("div", {"class": "metadata-description"}) + assert ( + 'An example SQLite database demonstrating Datasette. Sign in as root user' + == inner_html(soup.find("div", {"class": "metadata-description"})) ) assert_footer_links(soup) @@ -1451,6 +1452,7 @@ def test_base_url_config(app_client_base_url_prefix, path): "https://github.com/simonw/datasette", "https://github.com/simonw/datasette/blob/master/LICENSE", "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", + "/login-as-root", # Only used for the latest.datasette.io demo } and not href.startswith("https://plugin-example.com/") ): From b84cfe1b08ec3a881767e30122b7d4c0fa03f9e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 10:40:09 -0700 Subject: [PATCH 0142/1866] Confirm table actions work on views, closes #1067 --- tests/test_plugins.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index be36a517..6a4ea60a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -784,7 +784,8 @@ def test_hook_menu_links(app_client): ] -def test_hook_table_actions(app_client): +@pytest.mark.parametrize("table_or_view", ["facetable", "simple_view"]) +def test_hook_table_actions(app_client, table_or_view): def get_table_actions_links(html): soup = Soup(html, "html.parser") details = soup.find("details", {"class": "table-menu-links"}) @@ -792,12 +793,12 @@ def test_hook_table_actions(app_client): return [] return [{"label": a.text, "href": a["href"]} for a in details.select("a")] - response = app_client.get("/fixtures/facetable") + response = app_client.get("/fixtures/{}".format(table_or_view)) assert get_table_actions_links(response.text) == [] - response_2 = app_client.get("/fixtures/facetable?_bot=1") + response_2 = app_client.get("/fixtures/{}?_bot=1".format(table_or_view)) assert get_table_actions_links(response_2.text) == [ {"label": "From async", "href": "/"}, {"label": "Database: fixtures", "href": "/"}, - {"label": "Table: facetable", "href": "/"}, + {"label": "Table: {}".format(table_or_view), "href": "/"}, ] From 11eb1e026f3d84cb771f8d6e204939cbaee130cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 11:16:28 -0700 Subject: [PATCH 0143/1866] datasette.urls.table(..., format="json"), closes #1035 Also improved tests for datasette.urls and added format= to some other methods --- datasette/app.py | 42 +++++++++++++++++++++++++----------- datasette/utils/__init__.py | 10 +++++---- datasette/views/base.py | 10 +++++---- datasette/views/database.py | 4 ++-- docs/internals.rst | 14 ++++++------ tests/test_internals_urls.py | 42 +++++++++++++++++++++++++++++------- tests/test_utils.py | 10 ++++++--- 7 files changed, 92 insertions(+), 40 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8cff6577..3a06d911 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -53,6 +53,7 @@ from .utils import ( format_bytes, module_from_path, parse_metadata, + path_with_format, resolve_env_secrets, sqlite3, to_css_class, @@ -1285,13 +1286,16 @@ class Urls: def __init__(self, ds): self.ds = ds - def path(self, path): + def path(self, path, format=None): if path.startswith("/"): path = path[1:] - return self.ds.config("base_url") + path + path = self.ds.config("base_url") + path + if format is not None: + path = path_with_format(path=path, format=format) + return path - def instance(self): - return self.path("") + def instance(self, format=None): + return self.path("", format=format) def static(self, path): return self.path("-/static/{}".format(path)) @@ -1302,21 +1306,33 @@ class Urls: def logout(self): return self.path("-/logout") - def database(self, database): + def database(self, database, format=None): db = self.ds.databases[database] if self.ds.config("hash_urls") and db.hash: - return self.path("{}-{}".format(database, db.hash[:HASH_LENGTH])) + path = self.path( + "{}-{}".format(database, db.hash[:HASH_LENGTH]), format=format + ) else: - return self.path(database) + path = self.path(database, format=format) + return path - def table(self, database, table): - return "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) + def table(self, database, table, format=None): + path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) + if format is not None: + path = path_with_format(path=path, format=format) + return path - def query(self, database, query): - return "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) + def query(self, database, query, format=None): + path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) + if format is not None: + path = path_with_format(path=path, format=format) + return path - def row(self, database, table, row_path): - return "{}/{}".format(self.table(database, table), row_path) + def row(self, database, table, row_path, format=None): + path = "{}/{}".format(self.table(database, table), row_path) + if format is not None: + path = path_with_format(path=path, format=format) + return path def row_blob(self, database, table, row_path, column): return self.table(database, table) + "/{}.blob?_blob_column={}".format( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 33decbfc..bf361784 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -678,9 +678,11 @@ async def resolve_table_and_format( return table_and_format, None -def path_with_format(request, format, extra_qs=None, replace_format=None): +def path_with_format( + *, request=None, path=None, format=None, extra_qs=None, replace_format=None +): qs = extra_qs or {} - path = request.path + path = request.path if request else path if replace_format and path.endswith(".{}".format(replace_format)): path = path[: -(1 + len(replace_format))] if "." in path: @@ -689,11 +691,11 @@ def path_with_format(request, format, extra_qs=None, replace_format=None): path = "{}.{}".format(path, format) if qs: extra = urllib.parse.urlencode(sorted(qs.items())) - if request.query_string: + if request and request.query_string: path = "{}?{}&{}".format(path, request.query_string, extra) else: path = "{}?{}".format(path, extra) - elif request.query_string: + elif request and request.query_string: path = "{}?{}".format(path, request.query_string) return path diff --git a/datasette/views/base.py b/datasette/views/base.py index 6ca78934..430489c1 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -333,8 +333,8 @@ class DataView(BaseView): cell = self.ds.absolute_url( request, path_with_format( - request, - "blob", + request=request, + format="blob", extra_qs={ "_blob_column": column, "_blob_hash": hashlib.sha256( @@ -535,11 +535,13 @@ class DataView(BaseView): it_can_render = await await_me_maybe(it_can_render) if it_can_render: renderers[key] = path_with_format( - request, key, {**url_labels_extra} + request=request, format=key, extra_qs={**url_labels_extra} ) url_csv_args = {"_size": "max", **url_labels_extra} - url_csv = path_with_format(request, "csv", url_csv_args) + url_csv = path_with_format( + request=request, format="csv", extra_qs=url_csv_args + ) url_csv_path = url_csv.split("?")[0] context = { **data, diff --git a/datasette/views/database.py b/datasette/views/database.py index 8b9e8833..3ed60f4e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -346,8 +346,8 @@ class QueryView(DataView): ) elif isinstance(display_value, bytes): blob_url = path_with_format( - request, - "blob", + request=request, + format="blob", extra_qs={ "_blob_column": column, "_blob_hash": hashlib.sha256( diff --git a/docs/internals.rst b/docs/internals.rst index 4ebeb983..ee7fe6e4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -396,10 +396,10 @@ datasette.urls The ``datasette.urls`` object contains methods for building URLs to pages within Datasette. Plugins should use this to link to pages, since these methods take into account any :ref:`config_base_url` configuration setting that might be in effect. -``datasette.urls.instance()`` - Returns the URL to the Datasette instance root page. This is usually ``"/"`` +``datasette.urls.instance(format=None)`` + Returns the URL to the Datasette instance root page. This is usually ``"/"``. -``datasette.urls.path(path)`` +``datasette.urls.path(path, format=None)`` Takes a path and returns the full path, taking ``base_url`` into account. For example, ``datasette.urls.path("-/logout")`` will return the path to the logout page, which will be ``"/-/logout"`` by default or ``/prefix-path/-/logout`` if ``base_url`` is set to ``/prefix-path/`` @@ -423,13 +423,13 @@ The ``datasette.urls`` object contains methods for building URLs to pages within ``datasette.url.static_plugins("datasette_cluster_map", "datasette-cluster-map.js")`` would return ``"/-/static-plugins/datasette_cluster_map/datasette-cluster-map.js"`` -``datasette.urls.database(database_name)`` +``datasette.urls.database(database_name, format=None)`` Returns the URL to a database page, for example ``"/fixtures"`` -``datasette.urls.table(database_name, table_name)`` +``datasette.urls.table(database_name, table_name, format=None)`` Returns the URL to a table page, for example ``"/fixtures/facetable"`` -``datasette.urls.query(database_name, query_name)`` +``datasette.urls.query(database_name, query_name, format=None)`` Returns the URL to a query page, for example ``"/fixtures/pragma_cache_size"`` These functions can be accessed via the ``{{ urls }}`` object in Datasette templates, for example: @@ -441,6 +441,8 @@ These functions can be accessed via the ``{{ urls }}`` object in Datasette templ facetable table pragma_cache_size query +Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is usually the path with ``.json`` added on the end, but it may use ``?_format=json`` in cases where the path already includes ``.json``, for example a URL to a table named ``table.json``. + .. _internals_database: Database class diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 6498ee43..005903df 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -82,18 +82,44 @@ def test_logout(ds, base_url, expected): @pytest.mark.parametrize( - "base_url,expected", + "base_url,format,expected", [ - ("/", "/:memory:"), - ("/prefix/", "/prefix/:memory:"), + ("/", None, "/:memory:"), + ("/prefix/", None, "/prefix/:memory:"), + ("/", "json", "/:memory:.json"), ], ) -def test_database(ds, base_url, expected): +def test_database(ds, base_url, format, expected): ds._config["base_url"] = base_url - assert ds.urls.database(":memory:") == expected - # Do table and query while we are here - assert ds.urls.table(":memory:", "name") == expected + "/name" - assert ds.urls.query(":memory:", "name") == expected + "/name" + assert ds.urls.database(":memory:", format=format) == expected + + +@pytest.mark.parametrize( + "base_url,name,format,expected", + [ + ("/", "name", None, "/:memory:/name"), + ("/prefix/", "name", None, "/prefix/:memory:/name"), + ("/", "name", "json", "/:memory:/name.json"), + ("/", "name.json", "json", "/:memory:/name.json?_format=json"), + ], +) +def test_table_and_query(ds, base_url, name, format, expected): + ds._config["base_url"] = base_url + assert ds.urls.table(":memory:", name, format=format) == expected + assert ds.urls.query(":memory:", name, format=format) == expected + + +@pytest.mark.parametrize( + "base_url,format,expected", + [ + ("/", None, "/:memory:/facetable/1"), + ("/prefix/", None, "/prefix/:memory:/facetable/1"), + ("/", "json", "/:memory:/facetable/1.json"), + ], +) +def test_row(ds, base_url, format, expected): + ds._config["base_url"] = base_url + assert ds.urls.row(":memory:", "facetable", "1", format=format) == expected @pytest.mark.parametrize("base_url", ["/", "/prefix/"]) diff --git a/tests/test_utils.py b/tests/test_utils.py index bae3b685..2d2ff52d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -382,15 +382,19 @@ def test_table_columns(): ) def test_path_with_format(path, format, extra_qs, expected): request = Request.fake(path) - actual = utils.path_with_format(request, format, extra_qs) + actual = utils.path_with_format(request=request, format=format, extra_qs=extra_qs) assert expected == actual def test_path_with_format_replace_format(): request = Request.fake("/foo/bar.csv") - assert utils.path_with_format(request, "blob") == "/foo/bar.csv?_format=blob" assert ( - utils.path_with_format(request, "blob", replace_format="csv") == "/foo/bar.blob" + utils.path_with_format(request=request, format="blob") + == "/foo/bar.csv?_format=blob" + ) + assert ( + utils.path_with_format(request=request, format="blob", replace_format="csv") + == "/foo/bar.blob" ) From c1d386ef67786f07d69e566b8e054e92949a844f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 11:43:36 -0700 Subject: [PATCH 0144/1866] Refactor Urls into url_builder.py Refs #1026 --- datasette/app.py | 60 +--------------------------------------- datasette/url_builder.py | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 59 deletions(-) create mode 100644 datasette/url_builder.py diff --git a/datasette/app.py b/datasette/app.py index 3a06d911..860f4563 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -40,6 +40,7 @@ from .views.special import ( ) from .views.table import RowView, TableView from .renderer import json_renderer +from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( @@ -53,7 +54,6 @@ from .utils import ( format_bytes, module_from_path, parse_metadata, - path_with_format, resolve_env_secrets, sqlite3, to_css_class, @@ -1280,61 +1280,3 @@ class DatasetteClient: async def request(self, method, path, **kwargs): async with httpx.AsyncClient(app=self.app) as client: return await client.request(method, self._fix(path), **kwargs) - - -class Urls: - def __init__(self, ds): - self.ds = ds - - def path(self, path, format=None): - if path.startswith("/"): - path = path[1:] - path = self.ds.config("base_url") + path - if format is not None: - path = path_with_format(path=path, format=format) - return path - - def instance(self, format=None): - return self.path("", format=format) - - def static(self, path): - return self.path("-/static/{}".format(path)) - - def static_plugins(self, plugin, path): - return self.path("-/static-plugins/{}/{}".format(plugin, path)) - - def logout(self): - return self.path("-/logout") - - def database(self, database, format=None): - db = self.ds.databases[database] - if self.ds.config("hash_urls") and db.hash: - path = self.path( - "{}-{}".format(database, db.hash[:HASH_LENGTH]), format=format - ) - else: - path = self.path(database, format=format) - return path - - def table(self, database, table, format=None): - path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) - if format is not None: - path = path_with_format(path=path, format=format) - return path - - def query(self, database, query, format=None): - path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) - if format is not None: - path = path_with_format(path=path, format=format) - return path - - def row(self, database, table, row_path, format=None): - path = "{}/{}".format(self.table(database, table), row_path) - if format is not None: - path = path_with_format(path=path, format=format) - return path - - def row_blob(self, database, table, row_path, column): - return self.table(database, table) + "/{}.blob?_blob_column={}".format( - row_path, urllib.parse.quote_plus(column) - ) diff --git a/datasette/url_builder.py b/datasette/url_builder.py new file mode 100644 index 00000000..c1bf629b --- /dev/null +++ b/datasette/url_builder.py @@ -0,0 +1,60 @@ +from .utils import path_with_format, HASH_LENGTH +import urllib + + +class Urls: + def __init__(self, ds): + self.ds = ds + + def path(self, path, format=None): + if path.startswith("/"): + path = path[1:] + path = self.ds.config("base_url") + path + if format is not None: + path = path_with_format(path=path, format=format) + return path + + def instance(self, format=None): + return self.path("", format=format) + + def static(self, path): + return self.path("-/static/{}".format(path)) + + def static_plugins(self, plugin, path): + return self.path("-/static-plugins/{}/{}".format(plugin, path)) + + def logout(self): + return self.path("-/logout") + + def database(self, database, format=None): + db = self.ds.databases[database] + if self.ds.config("hash_urls") and db.hash: + path = self.path( + "{}-{}".format(database, db.hash[:HASH_LENGTH]), format=format + ) + else: + path = self.path(database, format=format) + return path + + def table(self, database, table, format=None): + path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) + if format is not None: + path = path_with_format(path=path, format=format) + return path + + def query(self, database, query, format=None): + path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) + if format is not None: + path = path_with_format(path=path, format=format) + return path + + def row(self, database, table, row_path, format=None): + path = "{}/{}".format(self.table(database, table), row_path) + if format is not None: + path = path_with_format(path=path, format=format) + return path + + def row_blob(self, database, table, row_path, column): + return self.table(database, table) + "/{}.blob?_blob_column={}".format( + row_path, urllib.parse.quote_plus(column) + ) From 7a67bc7a569509d65b3a8661e0ad2c65f0b09166 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 12:11:40 -0700 Subject: [PATCH 0145/1866] datasette.urls methods will not apply base_url prefix twice, refs #1026 --- datasette/url_builder.py | 17 +++++++------- datasette/utils/__init__.py | 23 ++++++++++++++++++ docs/internals.rst | 2 ++ tests/test_internals_urls.py | 45 ++++++++++++++++++++++++++++-------- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index c1bf629b..bcc4f39d 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import path_with_format, HASH_LENGTH +from .utils import path_with_format, HASH_LENGTH, PrefixedUrlString import urllib @@ -7,12 +7,13 @@ class Urls: self.ds = ds def path(self, path, format=None): - if path.startswith("/"): - path = path[1:] - path = self.ds.config("base_url") + path + if not isinstance(path, PrefixedUrlString): + if path.startswith("/"): + path = path[1:] + path = self.ds.config("base_url") + path if format is not None: path = path_with_format(path=path, format=format) - return path + return PrefixedUrlString(path) def instance(self, format=None): return self.path("", format=format) @@ -40,19 +41,19 @@ class Urls: path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) if format is not None: path = path_with_format(path=path, format=format) - return path + return PrefixedUrlString(path) def query(self, database, query, format=None): path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) if format is not None: path = path_with_format(path=path, format=format) - return path + return PrefixedUrlString(path) def row(self, database, table, row_path, format=None): path = "{}/{}".format(self.table(database, table), row_path) if format is not None: path = path_with_format(path=path, format=format) - return path + return PrefixedUrlString(path) def row_blob(self, database, table, row_path, column): return self.table(database, table) + "/{}.blob?_blob_column={}".format( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bf361784..21fa944c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1010,3 +1010,26 @@ async def initial_path_for_datasette(datasette): else: path = datasette.urls.instance() return path + + +class PrefixedUrlString(str): + def __add__(self, other): + return type(self)(super().__add__(other)) + + def __getattribute__(self, name): + if name in dir(str): + + def method(self, *args, **kwargs): + value = getattr(super(), name)(*args, **kwargs) + if isinstance(value, str): + return type(self)(value) + elif isinstance(value, list): + return [type(self)(i) for i in value] + elif isinstance(value, tuple): + return tuple(type(self)(i) for i in value) + else: + return value + + return method.__get__(self) + else: + return super().__getattribute__(name) diff --git a/docs/internals.rst b/docs/internals.rst index ee7fe6e4..8594e36a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -443,6 +443,8 @@ These functions can be accessed via the ``{{ urls }}`` object in Datasette templ Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) arguments to get back URLs to the JSON representation. This is usually the path with ``.json`` added on the end, but it may use ``?_format=json`` in cases where the path already includes ``.json``, for example a URL to a table named ``table.json``. +These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path. + .. _internals_database: Database class diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 005903df..a56d735b 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -1,4 +1,5 @@ from datasette.app import Datasette +from datasette.utils import PrefixedUrlString from .fixtures import app_client_with_hash import pytest @@ -20,7 +21,17 @@ def ds(): ) def test_path(ds, base_url, path, expected): ds._config["base_url"] = base_url - assert ds.urls.path(path) == expected + actual = ds.urls.path(path) + assert actual == expected + assert isinstance(actual, PrefixedUrlString) + + +def test_path_applied_twice_does_not_double_prefix(ds): + ds._config["base_url"] = "/prefix/" + path = ds.urls.path("/") + assert path == "/prefix/" + path = ds.urls.path(path) + assert path == "/prefix/" @pytest.mark.parametrize( @@ -32,7 +43,9 @@ def test_path(ds, base_url, path, expected): ) def test_instance(ds, base_url, expected): ds._config["base_url"] = base_url - assert ds.urls.instance() == expected + actual = ds.urls.instance() + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize( @@ -44,7 +57,9 @@ def test_instance(ds, base_url, expected): ) def test_static(ds, base_url, file, expected): ds._config["base_url"] = base_url - assert ds.urls.static(file) == expected + actual = ds.urls.static(file) + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize( @@ -66,7 +81,9 @@ def test_static(ds, base_url, file, expected): ) def test_static_plugins(ds, base_url, plugin, file, expected): ds._config["base_url"] = base_url - assert ds.urls.static_plugins(plugin, file) == expected + actual = ds.urls.static_plugins(plugin, file) + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize( @@ -78,7 +95,9 @@ def test_static_plugins(ds, base_url, plugin, file, expected): ) def test_logout(ds, base_url, expected): ds._config["base_url"] = base_url - assert ds.urls.logout() == expected + actual = ds.urls.logout() + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize( @@ -91,7 +110,9 @@ def test_logout(ds, base_url, expected): ) def test_database(ds, base_url, format, expected): ds._config["base_url"] = base_url - assert ds.urls.database(":memory:", format=format) == expected + actual = ds.urls.database(":memory:", format=format) + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize( @@ -105,8 +126,12 @@ def test_database(ds, base_url, format, expected): ) def test_table_and_query(ds, base_url, name, format, expected): ds._config["base_url"] = base_url - assert ds.urls.table(":memory:", name, format=format) == expected - assert ds.urls.query(":memory:", name, format=format) == expected + actual1 = ds.urls.table(":memory:", name, format=format) + assert actual1 == expected + assert isinstance(actual1, PrefixedUrlString) + actual2 = ds.urls.query(":memory:", name, format=format) + assert actual2 == expected + assert isinstance(actual2, PrefixedUrlString) @pytest.mark.parametrize( @@ -119,7 +144,9 @@ def test_table_and_query(ds, base_url, name, format, expected): ) def test_row(ds, base_url, format, expected): ds._config["base_url"] = base_url - assert ds.urls.row(":memory:", "facetable", "1", format=format) == expected + actual = ds.urls.row(":memory:", "facetable", "1", format=format) + assert actual == expected + assert isinstance(actual, PrefixedUrlString) @pytest.mark.parametrize("base_url", ["/", "/prefix/"]) From 84bc7244c106ab6175b8315a2d917cf29ea53c4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 12:29:42 -0700 Subject: [PATCH 0146/1866] datasette.client now applies base_url, closes #1026 --- datasette/app.py | 4 +++ datasette/utils/asgi.py | 4 +-- docs/internals.rst | 12 +++++++ tests/plugins/my_plugin.py | 4 +++ tests/test_internals_datasette_client.py | 45 ++++++++++++++++++------ 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 860f4563..8db650e9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -44,6 +44,7 @@ from .url_builder import Urls from .database import Database, QueryInterrupted from .utils import ( + PrefixedUrlString, async_call_with_supported_arguments, await_me_maybe, call_with_supported_arguments, @@ -1242,9 +1243,12 @@ class NotFoundExplicit(NotFound): class DatasetteClient: def __init__(self, ds): + self.ds = ds self.app = ds.app() def _fix(self, path): + if not isinstance(path, PrefixedUrlString): + path = self.ds.urls.path(path) if path.startswith("/"): path = "http://localhost{}".format(path) return path diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index f438f829..e4c8ce5c 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -387,9 +387,9 @@ class Response: ) @classmethod - def json(cls, body, status=200, headers=None): + def json(cls, body, status=200, headers=None, default=None): return cls( - json.dumps(body), + json.dumps(body, default=default), status=status, headers=headers, content_type="application/json; charset=utf-8", diff --git a/docs/internals.rst b/docs/internals.rst index 8594e36a..d3d0be8e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -387,6 +387,18 @@ It offers the following methods: ``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response Execute an internal request with the given HTTP method against that path. +These methods can be used with :ref:`internals_datasette_urls` - for example: + +.. code-block:: python + + table_json = ( + await datasette.client.get( + datasette.urls.table("fixtures", "facetable", format="json") + ) + ).json() + +``datasette.client`` methods automatically take the current :ref:`config_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path. + For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. .. _internals_datasette_urls: diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index b487cdf0..767c363d 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -257,6 +257,9 @@ def register_routes(): ) ) + def asgi_scope(scope): + return Response.json(scope, default=repr) + return [ (r"/one/$", one), (r"/two/(?P.*)$", two), @@ -267,6 +270,7 @@ def register_routes(): (r"/not-async/$", not_async), (r"/add-message/$", add_message), (r"/render-message/$", render_message), + (r"/asgi-scope$", asgi_scope), ] diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index d73fbb06..0b1c5f0e 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -31,14 +31,37 @@ async def test_client_methods(datasette, method, path, expected_status): @pytest.mark.asyncio -async def test_client_post(datasette): - response = await datasette.client.post( - "/-/messages", - data={ - "message": "A message", - }, - allow_redirects=False, - ) - assert isinstance(response, httpx.Response) - assert response.status_code == 302 - assert "ds_messages" in response.cookies +@pytest.mark.parametrize("prefix", [None, "/prefix/"]) +async def test_client_post(datasette, prefix): + original_base_url = datasette._config["base_url"] + try: + if prefix is not None: + datasette._config["base_url"] = prefix + response = await datasette.client.post( + "/-/messages", + data={ + "message": "A message", + }, + allow_redirects=False, + ) + assert isinstance(response, httpx.Response) + assert response.status_code == 302 + assert "ds_messages" in response.cookies + finally: + datasette._config["base_url"] = original_base_url + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prefix,expected_path", [(None, "/asgi-scope"), ("/prefix/", "/prefix/asgi-scope")] +) +async def test_client_path(datasette, prefix, expected_path): + original_base_url = datasette._config["base_url"] + try: + if prefix is not None: + datasette._config["base_url"] = prefix + response = await datasette.client.get("/asgi-scope") + path = response.json()["path"] + assert path == expected_path + finally: + datasette._config["base_url"] = original_base_url From bf18b9ba175a7b25fb8b765847397dd6efb8bb7b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 12:47:42 -0700 Subject: [PATCH 0147/1866] Stop using plugin-example.com, closes #1074 --- tests/plugins/my_plugin.py | 6 +++--- tests/plugins/my_plugin_2.py | 4 ++-- tests/test_html.py | 2 +- tests/test_plugins.py | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 767c363d..cd2c8e23 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -29,7 +29,7 @@ def prepare_connection(conn, database, datasette): def extra_css_urls(template, database, table, view_name, columns, request, datasette): async def inner(): return [ - "https://plugin-example.com/{}/extra-css-urls-demo.css".format( + "https://plugin-example.datasette.io/{}/extra-css-urls-demo.css".format( base64.b64encode( json.dumps( { @@ -57,10 +57,10 @@ def extra_css_urls(template, database, table, view_name, columns, request, datas def extra_js_urls(): return [ { - "url": "https://plugin-example.com/jquery.js", + "url": "https://plugin-example.datasette.io/jquery.js", "sri": "SRIHASH", }, - "https://plugin-example.com/plugin1.js", + "https://plugin-example.datasette.io/plugin1.js", ] diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 7d8095ed..6cd222e6 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -8,10 +8,10 @@ import json def extra_js_urls(): return [ { - "url": "https://plugin-example.com/jquery.js", + "url": "https://plugin-example.datasette.io/jquery.js", "sri": "SRIHASH", }, - "https://plugin-example.com/plugin2.js", + "https://plugin-example.datasette.io/plugin2.js", ] diff --git a/tests/test_html.py b/tests/test_html.py index 7c068085..79b6138d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1454,7 +1454,7 @@ def test_base_url_config(app_client_base_url_prefix, path): "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", "/login-as-root", # Only used for the latest.datasette.io demo } - and not href.startswith("https://plugin-example.com/") + and not href.startswith("https://plugin-example.datasette.io/") ): # If this has been made absolute it may start http://localhost/ if href.startswith("http://localhost/"): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 6a4ea60a..5e3d6dc3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -125,7 +125,7 @@ def test_hook_extra_js_urls(app_client): == { "integrity": "SRIHASH", "crossorigin": "anonymous", - "src": "https://plugin-example.com/jquery.js", + "src": "https://plugin-example.datasette.io/jquery.js", } ] @@ -135,7 +135,7 @@ def test_plugins_with_duplicate_js_urls(app_client): response = app_client.get("/fixtures") # This test is a little tricky, as if the user has any other plugins in # their current virtual environment those may affect what comes back too. - # What matters is that https://plugin-example.com/jquery.js is only there once + # What matters is that https://plugin-example.datasette.io/jquery.js is only there once # and it comes before plugin1.js and plugin2.js which could be in either # order scripts = Soup(response.body, "html.parser").findAll("script") @@ -143,16 +143,16 @@ def test_plugins_with_duplicate_js_urls(app_client): # No duplicates allowed: assert len(srcs) == len(set(srcs)) # jquery.js loaded once: - assert 1 == srcs.count("https://plugin-example.com/jquery.js") + assert 1 == srcs.count("https://plugin-example.datasette.io/jquery.js") # plugin1.js and plugin2.js are both there: - assert 1 == srcs.count("https://plugin-example.com/plugin1.js") - assert 1 == srcs.count("https://plugin-example.com/plugin2.js") + assert 1 == srcs.count("https://plugin-example.datasette.io/plugin1.js") + assert 1 == srcs.count("https://plugin-example.datasette.io/plugin2.js") # jquery comes before them both - assert srcs.index("https://plugin-example.com/jquery.js") < srcs.index( - "https://plugin-example.com/plugin1.js" + assert srcs.index("https://plugin-example.datasette.io/jquery.js") < srcs.index( + "https://plugin-example.datasette.io/plugin1.js" ) - assert srcs.index("https://plugin-example.com/jquery.js") < srcs.index( - "https://plugin-example.com/plugin2.js" + assert srcs.index("https://plugin-example.datasette.io/jquery.js") < srcs.index( + "https://plugin-example.datasette.io/plugin2.js" ) From a4ca26a2659d21779adf625183061d8879954c15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 13:35:47 -0700 Subject: [PATCH 0148/1866] Address PrefixedUrlString bug in #1075 --- datasette/app.py | 3 +++ datasette/utils/__init__.py | 7 +++++-- tests/fixtures.py | 1 + tests/test_api.py | 1 + tests/test_cli_serve_get.py | 1 + tests/test_html.py | 5 +++++ 6 files changed, 16 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8db650e9..1271e52f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -822,6 +822,9 @@ class Datasette: if url in seen_urls: continue seen_urls.add(url) + if url.startswith("/"): + # Take base_url into account: + url = self.urls.path(url) if sri: output.append({"url": url, "sri": sri}) else: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 21fa944c..a7d96401 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,8 +1,8 @@ import asyncio from contextlib import contextmanager +import click from collections import OrderedDict, namedtuple import base64 -import click import hashlib import inspect import itertools @@ -1016,8 +1016,11 @@ class PrefixedUrlString(str): def __add__(self, other): return type(self)(super().__add__(other)) + def __str__(self): + return super().__str__() + def __getattribute__(self, name): - if name in dir(str): + if not name.startswith("__") and name in dir(str): def method(self, *args, **kwargs): value = getattr(super(), name)(*args, **kwargs) diff --git a/tests/fixtures.py b/tests/fixtures.py index 5cbfc72f..d2ac661d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -274,6 +274,7 @@ METADATA = { "source_url": "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", "about": "About Datasette", "about_url": "https://github.com/simonw/datasette", + "extra_css_urls": ["/static/extra-css-urls.css"], "plugins": { "name-of-plugin": {"depth": "root"}, "env-plugin": {"foo": {"$env": "FOO_ENV"}}, diff --git a/tests/test_api.py b/tests/test_api.py index 5e9c1a0a..18e4b9e4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1852,6 +1852,7 @@ def test_paginate_using_link_header(app_client, qs): num_pages = 0 while path: response = app_client.get(path) + assert response.status == 200 num_pages += 1 link = response.headers.get("link") if link: diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 8f1665a9..39236dd8 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -61,6 +61,7 @@ def test_serve_with_get_exit_code_for_error(tmp_path_factory): "--get", "/this-is-404", ], + catch_exceptions=False, ) assert result.exit_code == 1 assert "404" in result.output diff --git a/tests/test_html.py b/tests/test_html.py index 79b6138d..006c223d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1466,6 +1466,11 @@ def test_base_url_config(app_client_base_url_prefix, path): } +def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): + html = app_client_base_url_prefix.get("/").text + assert '' in html + + @pytest.mark.parametrize( "path,expected", [ From 6bb41c4b33dbd1015c181cd43465b645298c3c88 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 13:48:39 -0700 Subject: [PATCH 0149/1866] Fix for test_paginate_using_link_header --- tests/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_api.py b/tests/test_api.py index 18e4b9e4..3365bf57 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1859,6 +1859,7 @@ def test_paginate_using_link_header(app_client, qs): assert link.startswith("<") assert link.endswith('>; rel="next"') path = link[1:].split(">")[0] + path = path.replace("http://localhost", "") else: path = None assert num_pages == 21 From 1fe15f4dc110622754d9dbeafe0f93c79fde9022 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 14:13:57 -0700 Subject: [PATCH 0150/1866] Docs: Running Datasette behind a proxy, closes #1027 --- docs/deploying.rst | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index b0647b2f..e777f296 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -110,3 +110,57 @@ If you want to build SQLite files or download them as part of the deployment pro wget https://fivethirtyeight.datasettes.com/fivethirtyeight.db `simonw/buildpack-datasette-demo `__ is an example GitHub repository showing a simple Datasette configuration that can be deployed to a buildpack-supporting host. + +.. _deploying_proxy: + +Running Datasette behind a proxy +================================ + +You may wish to run Datasette behind an Apache or nginx proxy, using a path within your existing site. + +You can use the :ref:`config_base_url` configuration setting to tell Datasette to serve traffic with a specific URL prefix. For example, you could run Datasette like this:: + + datasette my-database.db --config base_url:/my-datasette/ -p 8009 + +This will run Datasette with the following URLs: + +- ``http://127.0.0.1:8009/my-datasette/`` - the Datasette homepage +- ``http://127.0.0.1:8009/my-datasette/my-database`` - the page for the ``my-database.db`` database +- ``http://127.0.0.1:8009/my-datasette/my-database/some_table`` - the page for the ``some_table`` table + +You can now set your nginx or Apache server to proxy the ``/my-datasette/`` path to this Datasette instance. + +Nginx proxy configuration +------------------------- + +Here is an example of an `nginx `__ configuration file that will proxy traffic to Datasette:: + + daemon off; + + events { + worker_connections 1024; + } + + http { + server { + listen 80; + + location /my-datasette { + proxy_pass http://127.0.0.1:8009; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + } + +Apache proxy configuration +-------------------------- + +For `Apache `__, you can use the ``ProxyPass`` directive. First make sure the following lines are uncommented:: + + LoadModule proxy_module lib/httpd/modules/mod_proxy.so + LoadModule proxy_http_module lib/httpd/modules/mod_proxy_http.so + +Then add this directive to proxy traffic:: + + ProxyPass /datasette-prefix/ http://127.0.0.1:8009/datasette-prefix/ From fa4de7551cbaf5e08f022d106605252d2a4332ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 14:37:58 -0700 Subject: [PATCH 0151/1866] Binary data documentation, closes #1047 --- docs/binary_data.png | Bin 0 -> 5572 bytes docs/binary_data.rst | 68 +++++++++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 2 +- docs/csv_export.rst | 2 +- docs/index.rst | 1 + 5 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 docs/binary_data.png create mode 100644 docs/binary_data.rst diff --git a/docs/binary_data.png b/docs/binary_data.png new file mode 100644 index 0000000000000000000000000000000000000000..2c5d0bdda89fc03e1632c9843ac9a11321e367bb GIT binary patch literal 5572 zcmd5=S5OlSmj&q^r9~p3f^-lR3B8DP5>QY~%5MMcGCXz<_>71br! zKYKSL%|FwnH2d@)FfcQ*&^`Why1nyjW@T%bywEl{QQ9_~Qq@&lTWg1UIfd!-w>#XJ947xNz3@XLk75EeM3Wi<(d#HG8 zVI*s9Mnl>)Nt-~+3_p8)qs#83rgrU0z1p=aDmG7Q zhjIElO8t5jS>4o@P~bd6>+~7;o|v^O>rNP3*`M>)DDm2eY6F-p6adzv3Ss?8F?Em_ z=#259kA}RT%4@mtivgqy2)_S$*N639(ia1ek6P%N4yCj~)6++6 zmsRE0FyzgKMW?8nn;*P=^;y9GrS_m+{6$29 ze3mm}zuNZDk@dii=(*u{#mBM%)|?1+UGZRm7LKA3H?Hp11{*vk{dfz#u6-EDC>^=D z5bMth732k#v#;gbbQ&L|nxrr3c0?JqJN_$}_MfoP`$lUmNU;rQ~2B3w|VAZ9SSY&Gu# z9A#g@CPmC*=qTfeW0$m5?>RP)p4mXEXf{5|{AzQ{;tOo>%h)^&v4OrT?WtWkEpMS> zJvkz?DCe~-pgvY_OC;Y`fHGK$At%Or8t$!1l0G5$D1}WTF&5gj#`kLUWpsxDc&z8AFi=lb4_2 zrL)?IJ&!58sZyZ2QuI6vEiC+}%{XKpU zJ0{7fPG(gK_-)8~P@4QkEkR~{vA`&yM0>l)<{hi>6t^4!Eh632m_h~qU7k#2uq|k^ z#Zie;V$FxUo}#kSJ5{yr7#&QTlhvx&DIEt~j>)u~!-Xi_ee20m%}~SnK5>Y1I0x`@ zsgEcRfE=xh&UdhLi`emp^s^-a1tCUO8`q~7mPt*vIQo#oOB!?dAj&`yuF@PZe@FsH z?9njucA4#CrJKru8~*HAvr!MiQ%U+?;vR|Yg;warKA)H5k%3ZP+#~(t34#4Cf1cS! z{ijAB_8je%T_3XOsvY~VrVn!E!&nA65D|0mCRi}560mx1&R6Y4Pn1lpl3|x{_5o0c zrOVyP635ZW{#QPI6goLcxJ2!*Evy){qoP{Ki}g(vQZnw!h+wD|(5WrC{yGoHimgK! z9~rSkyh#E_(ZpJV#I7n)`#JRo`to2}wJJ%Bv8tS4 z;1Oy!+mP%|57h;wEQJr?X&@?_qh-k(X*(Ho&m}_-4@I*247zY|=+7!dyK*uwp_O&l zSc)^-^qaAze;-@_60tN|?E?y4`s1rhT^T^qj_$;MjwF6~t8h46-QkpRDmRFyd|qx9+|QxU}@Y>-GxR<#G*Pc zB33JEo$rkvv=9B)R`{9JHadHz#W&J&q52WVLx1soc5|jhVy5DTfEp$oBAhKh(gKxS zRT#M{{2xP{y-LZ93vT(hInB)T{pt$>WFcHr28uS}6{0N`OIxgd(vic$4xQ41fKE=2 zDzMI*@cg|49*#g5dO;>wvxh5Rmvj)5Dnu!Yr}�?>toCDIo1_3+ z%I}?(HBnQqqG`_E9vo)ceiak?H}oTH4;d&$@lYO#S1uSKZ8N2;zWW7PwuRr-uFd_y z#ylOgPH%7b8y@wmd?$ap0yCWn*Rf}hDSnSXA<@Z9T_U^|WwC#ggjg_&YaseGFosJN z@LY%y*IukoZG~9Z20qH*B3= zQ2c`={6|PfxwkjT9Dw{&RZt%Hev$7|Hqy0mLs_sYAe2t7xAhnQST*z$vO@HPyd~0J z_=@6E+YlToIRp*~wX?GZt+U(^UEcUc?6FDG!gV`D%l{D00+OYx9bX(?=``xn!X@N! z!{d;k=Q9pArYK)K%5#;8^pi3J?V?z*SLBGPu`P2R%GgTZh8~f)TbPo1Lh@X_`B3t< z7oxf-7*1IJoc|_)jrzrNHT^bAVzITFnIollO;J^$J(@hv_X8#?)YtZvKGrvQ4aUD4 zf)h>nQ=gv?3plMs2-{E|+-Z*u>Z{MG22!q>+?u0qv>>-4vU52+J?NC!H!v=v3%@0+ z&_iHiScRFmK#ZlGb*mR)+j>aGv*Fm5u4)`ilpg{oEU^9211V1}wMM+UHT(%h1J)Yg z(Go61)lKER?z+c7Y5qbMg>u^F+;D&Q^0?S*<<}0+ILt|17o^^P#Rb5bGtFoN8bF}H z>@P<6LE$i8<+7!(ZahilWQVf`8ML4Y10{b@h3i4wT$Ezy5i?>)s*z``Z|3eTD~iZ^ z$cL=^+P^*lmFQ8eN}BXqxD#$Fc3W*{iOe_o4upQFi{fQCFcUr(ABlS?@O#DB;{-#J z`GZ}>>{~d1K_8e)H(a`CSLtDE3hQjMT$gy2Dq(SRy!6i5apYo{AK^~xRkOp-34bt* z!8aqHBh1anl`!v^siA?QTMPvO9&b|E%hx){VkQKGHsZSg1kF}!ui%JhCD)*I$+kDP zz9-@2{wdW67u>oI9)SSi<7zS1By~ntD$1v_2wqm{Ob`3IllgRZ&Ba2XWrD=$L(Uk~ zMjp_(aozW!h{ldy6*2Ff+ZR}O)IAETg=`$$_9JTu_f9S1=f3i9Q=fh|$d z1-=U)7!zAst;GNZ6~OUz-Zh69tJTh0j;)frqg9uzWt-JUWY9Q+hS$Ms^u&bX?nQ0L z=P0MD{!F>Z+;i^Lj{F?oQRg<*RXrS0*Q9^p)1&juUu+bM-wHpsdppBnsf-05BqD1h z73U9YE?L5sA%|MK3HnH$R^u)`ES%lF(pKf@rjzDSXk2zB4dnT9rYdFEH zyu=pI+h(hCCe(=nv$a-^#Q|m;tmm|hYNj0n%kvF-7w=Z>pMmDbWv5AFJ=NVy4L55)8#FtCh2qik$FvM++4c7D1 zCf=u=B90bF+UC(Fu?9t|RH}G6z1=Bu=lCb9eABiRJMjgt;_2!O_SU0^M;Y-m<)e1% z0}^x<4y9+Zm90_8YDiESNQ|p~*yT)H{W+Fm8#GfqYPUVmLu)S@rTM`7sJy{z-0k+! z^7HU$p6cAuff=UrGL@~RpJ{g*;YAVCCBg$mMyE?Z9pS{Tt~G2~Ww zSMUFc=_k~M5gk9)2I>+q&_h%RGJ2q+8IvmAgb8U??p$y+tE-tXD zp-UgdAxF|?5v}Bp>xPWfx+*C$+P#63PsidNZ#=;>@s04C1}4RX*pxTHPXLyy!0!WP zqS3Wcep-^!i_e66_T(s+4EVQ&^A%0_zp}7AX&kJwvfbtKX}lYlJtEd{`;}5-W7ojA`j*YjVl>dd zvYqSS%^4 z{1qTuoac{}ibU)6S{6jT9our$o9# zM4j6##tT=$gTLy|)6Mcv3?ac|9D5uK1QdApz8?+4=+Jxyx6KD*jYrNA`p?bnq-?&G z{5xVSeh4sszr=wk#zKP|G_?jEN*6BZ?av6s{cb5R%AajPoHzr|BO3Sdu zO(~nmMiFQ3cZl}fPIiTNl_&^4y-UhRXPNJGR4c$RplQrCDUppKvGOt&IagYQgH$9}iHvJ94ifqf>m-2bwY$Er$sCdUk6Vf2t3rp-ECaALtFZs)KWY+rDG)snGOYCgmQW5CGeO#?-W zmzhSpdAl1^ZIV(j$_`=T;!mCXr5$Nu`;VZ(`V)(9Ag6!dS0G0>{=f`8?L2_;g^Gge z$qWl%3A#Xh9{1|P^0M?S{WlZAgGozkhg*C5 z2h{Zw=L`eQE7nKH@n2ZgYYqf%@F`cOaNfqiR_j_`K{;?|zR8nRMc`^YDIu4i}^>2mW~i8u74>5p9f)O zL|z2?;qK^Ol4bz;16`dz8at<-55+b5q6^TZlf)nvl48<_MLl3V))N#v%^k45KySHd z;@x+>aBS>pb6?>OhEpRB7&y;0`^0~!+4$R<>_SxMw7CCS7{i~KAl=Q>uJn@39H4x= zv^SSa>~vka+g9@g)A0lExE<@r&~;OaIIOx{4*Z5|%Iv2Es`#oe@ij|-{uit|hAn6z zV-TJ>t>BcoVe4_*i7f)`!^-b0C}KB@;li%@J{9G~-%uT_(vER>?7C@lSLTPh=k$PY zAHK7du!;I`r~a{knj3VLbR}cp99{zPQ=r#8ifoURhn-v5Wz0 zW(*x1%6c*>xxY3~O=8Pq{Pcd}<*?fL1V30~GO2nx?B+QJzx5Xwc`)KyujRE;qmzwJ z-Q|ZaJeYcqL?m+I12-HH(hQ(0=&TYWYUqC}7WFX!`jRh${J?i=@OU zKRnp!dY}|mV0+Oc9`{rAo`>jCeGv1Gweps?m}c#+g_t-0M$A{moON0VR6u^B(>%M` ztyH*^ge>M#N`__. + +That page links to the binary value downloads. Those links look like this: + +https://latest.datasette.io/fixtures.blob?sql=select+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d + +These ``.blob`` links are also returned in the ``.csv`` exports Datasette provides for binary tables and queries, since the CSV format does not have a mechanism for representing binary data. + +Binary plugins +-------------- + +Several Datasette plugins are available that change the way Datasette treats binary data. + +- `datasette-render-binary `__ modifies +- https://github.com/simonw/datasette-render-images +- https://github.com/simonw/datasette-media \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 262400c8..fc566a37 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 0.51a2 (2020-10-30) ------------------- -- New :ref:`plugin_hook_load_template` plugin hook. (`#1042 `__) +- New ``load_template`` plugin hook. (`#1042 `__) - New :ref:`permissions_debug_menu` permission. (`#1068 `__) .. _v0_51_a1: diff --git a/docs/csv_export.rst b/docs/csv_export.rst index 9b7f8188..b5cc599a 100644 --- a/docs/csv_export.rst +++ b/docs/csv_export.rst @@ -1,6 +1,6 @@ .. _csv_export: -CSV Export +CSV export ========== Any Datasette table, view or custom SQL query can be exported as CSV. diff --git a/docs/index.rst b/docs/index.rst index 9096efd9..6b55da8c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,6 +46,7 @@ Contents authentication performance csv_export + binary_data facets full_text_search spatialite From d53d747e6a9dbc294c0565bc5eefe9aa16989316 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 15:13:39 -0700 Subject: [PATCH 0152/1866] Release 0.51 Refs #1014, #1016, #1019, #1023, #1027, #1028, #1033, #1034, #1036, #1039 Closes #1076 --- README.md | 1 + datasette/version.py | 2 +- docs/changelog.rst | 82 +++++++++++++++++++++++++--------------- docs/datasette-0.51.png | Bin 0 -> 47637 bytes 4 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 docs/datasette-0.51.png diff --git a/README.md b/README.md index 8670936c..c101a4ed 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly new ## News + * 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. diff --git a/datasette/version.py b/datasette/version.py index 2f4bc37e..f6e9ce97 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.51a2" +__version__ = "0.51" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index fc566a37..b9120c52 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,45 +4,65 @@ Changelog ========= -.. _v0_51_a2: +.. _v0_51: -0.51a2 (2020-10-30) -------------------- +0.51 (2020-10-31) +----------------- -- New ``load_template`` plugin hook. (`#1042 `__) +A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. + +New visual design +~~~~~~~~~~~~~~~~~ + +Datasette is no longer white and grey with blue and purple links! `Natalie Downe `__ has been working on a visual refresh, the first iteration of which is included in this release. (`#1056 `__) + +.. image:: datasette-0.51.png + :width: 740px + :alt: Screenshot showing Datasette's new visual look + +Plugins can now add links within Datasette +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A number of existing Datasette plugins add new pages to the Datasette interface, providig tools for things like `uploading CSVs `__, `editing table schemas `__ or `configuring full-text search `__. + +Plugins like this can now link to themselves from other parts of Datasette interface. The :ref:`plugin_hook_menu_links` hook (`#1064 `__) lets plugins add links to Datasette's new top-right application menu, and the :ref:`plugin_hook_table_actions` hook (`#1066 `__) adds links to a new "table actions" menu on the table page. + +The demo at `latest.datasette.io `__ now includes some example plugins. To see the new table actions menu first `sign into that demo as root `__ and then visit the `facetable `__ table to see the new cog icon menu at the top of the page. + +Binary data +~~~~~~~~~~~ + +SQLite tables can contain binary data in ``BLOB`` columns. Datasette now provides links for users to download this data directly from Datasette, and uses those links to make binary data available from CSV exports. See :ref:`binary` for more details. (`#1036 `__ and `#1034 `__). + +URL building +~~~~~~~~~~~~ + +The new :ref:`internals_datasette_urls` family of methods can be used to generate URLs to key pages within the Datasette interface, both within custom templates and Datasette plugins. See :ref:`writing_plugins_building_urls` for more details. (`#904 `__) + +Running Datasette behind a proxy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`config_base_url` configuration option is designed to help run Datasette on a specific path behind a proxy - for example if you want to run an instance of Datasette at ``/my-datasette/`` within your existing site's URL hierarchy, proxied behind nginx or Apache. + +Support for this configuration option has been greatly improved (`#1023 `__), and guidelines for using it are now available in a new documentation section on :ref:`deploying_proxy`. (`#1027 `__) + +Smaller changes +~~~~~~~~~~~~~~~ + +- Wide tables shown within Datasette now scroll horizontally (`#998 `__). This is achieved using a new ``
    `` element which may impact the implementation of some plugins (for example `this change to datasette-cluster-map `__). - New :ref:`permissions_debug_menu` permission. (`#1068 `__) - -.. _v0_51_a1: - -0.51a1 (2020-10-29) -------------------- - -- New colour scheme and improved visual design, courtesy of Natalie Downe. (`#1056 `__) -- scale-in animation for column action menu. (`#1039 `__) -- Wide tables now scroll horizontally. (`#998 `__) -- Option to pass a list of templates to ``.render_template()`` is now documented. (`#1045 `__) -- New ``datasette.urls.static_plugins()`` method. (`#1033 `__) -- ``BLOB`` column values can now be downloaded directly from the Datasette UI. (`#1036 `__) -- ``.csv`` exports now link to direct ``BLOB`` downloads. (`#1034 `__) -- ``datasette -o`` option now opens the most relevant page. (`#976 `__) -- ``datasette --cors`` option now enables access to ``/database.db`` downloads. (`#1057 `__) -- Database file downloads now implement cascading permissions, so you can download a database if you have ``view-database-download`` permission even if you do not have permission to access the Datasette instance. (`#1058 `__) -- New documentation on :ref:`writing_plugins_designing_urls`. (`#1053 `__) -- New navigation menu plus a :ref:`plugin_hook_menu_links` plugin hook to customize it. (`#1064 `__) -- :ref:`plugin_hook_table_actions` plugin hook for the new table actions menu. (`#1066 `__) - -.. _v0_51_a0: - -0.51a0 (2020-10-19) -------------------- - -- Fixed a bunch of issues relating to the :ref:`config_base_url` setting. (`#1023 `__) -- New ``datasette.urls`` URL builder for plugins, see :ref:`writing_plugins_building_urls`. (`#904 `__) - Removed ``--debug`` option, which didn't do anything. (`#814 `__) - ``Link:`` HTTP header pagination. (`#1014 `__) - ``x`` button for clearing filters. (`#1016 `__) - Edit SQL button on canned queries, (`#1019 `__) - ``--load-extension=spatialite`` shortcut. (`#1028 `__) +- scale-in animation for column action menu. (`#1039 `__) +- Option to pass a list of templates to ``.render_template()`` is now documented. (`#1045 `__) +- New ``datasette.urls.static_plugins()`` method. (`#1033 `__) +- ``datasette -o`` option now opens the most relevant page. (`#976 `__) +- ``datasette --cors`` option now enables access to ``/database.db`` downloads. (`#1057 `__) +- Database file downloads now implement cascading permissions, so you can download a database if you have ``view-database-download`` permission even if you do not have permission to access the Datasette instance. (`#1058 `__) +- New documentation on :ref:`writing_plugins_designing_urls`. (`#1053 `__) .. _v0_50_2: diff --git a/docs/datasette-0.51.png b/docs/datasette-0.51.png new file mode 100644 index 0000000000000000000000000000000000000000..eef83b9f6de49114baad50cfaac899f050559849 GIT binary patch literal 47637 zcmd?QWmFtdus=8i0)$`*9$W&!T?P$K(BKXMg8Sf3aAyX0f;$8cPC|ge9fA`WnBXwD z@8rF=pZ0(DytDgu_rreb)75qB-m3en>h7vr(H~Xiaj@QC0RRA;4+=6G0Kju70DvZr ziHfAGXtUiQFH#?swPc^3o^J1MuW#@6kB?_JHb=%LYQ`p8+kZeh`YI}_Kpll1d9B(t ziJ?(xx-sQkewX6jIiGFZ_Ic=8Y{24jhOfx@6Jv#10D!iH4>FQk-b)9ot4_;8_z{0N zu0v7KY0x(w*7vh^CNg&AJhyNhN{-z=suQ9z1FuDCn6Gr719Fz?umD2*Art`B1R9Ls zWO^vd1`HDf5CZ-WlT7N0Pb&0~lD&yK$`hmh{;sTEUp2n;oTE`&SmGjP@X@B`vL1lZ`0Xez4FwT``4fDA5d!-#;`oYpY>sJ0o zEI{3-vF&oY60h>8w07p!fccRF2P+96)B>|aINmEnbU>j3cjK+b_wqL!=UKW1BP}Ew zRhSFkeo<4vEFXMLz_BcnxEl|@Tp-d#%PF3-WEp}1+jTDx_hyK9{qNlD@@Ozlbj{D+NWwpeow4m5qf4 z4yIq4=Crk%wX9hTjyOHC^kdXVtDF4>J@I($CD-a4)9*iOiHrXdp8KiD9Bj>sV5szT zls7zO@8Y4%Oq15!(e_!)&ZUB-ZE3eG*~aKrRP!vl$3y1!7(8$DgNSQle@RvjWZcF1 zT-s*^yzj8CP;tJLWDzxktnM zLD$rzMvHtz{CZpiP6eyiF}$f;P>{d|*+>3PNrP+dN_4}p0H#U)4Tici%MIp%E$iE3 z<2+mXI_1NE6{@Tg^eyXP9yQHLUKH#ecI^qnwuc7Hu7tlI-YuXom#KwxDp@^-1!zee zpr#z=in zw%u!c*+A$@d5-qt$DIb;44kNAf%n`ePc7B`yQhIe(mPt*jUgJ&BCM;AYTfphVT-Rq==A($ldHfwLYK_=MGPtIo*HGgW4a`V|% z2MaJ?6vc^CyBzp=)y%?KK(DMZlyW0AS#Ljr({J!UtlT#$<%x`-AQxB%dg{=5bI~X= zeRV`%H|Pe#AZmNMJ~hm}A`vMHcmuP=8L(fc+)|Ez}h}QFJbpF4i-1Q2!mn$N7V?N)6}?9}O7% z6d$Dfm9?_O>7cOtei^R|7^*qX^I5DMD*qbpIn|JeC>bxJp>9=f`MGqqyspGzi-)-R zw74T%B|K5pu=HXB<93Fo?sayKMbX7qx??etip23{4eLs;P|=%KllVg4RqZ?JrdK`B z+xi)=vWJD!N6ndnsdp1YJMD~`sI*~4_KGcI{JX0(>4&LP84G1m=Xo4u>`-zg*Cfb+ zg9O@%8zn}Y57@5}_Pf}DH84O!x{HpwhT8ydeeaycl={uie&yq{Zif6t+lJYDZs^@N z#>DKbU#jJUDEFd0d%AkeI@DITu1!(%Wkl?kC)%v{7?Oa%a#JFJiyayekV7Gv5VY8l zz1rgX4p2nPEV&W#5~unqXs+T(k+oKTqBq*@iyIbUbO98xGxEGd;k}C#E`AOe6^#i7 z`0)bxKYhC*k50o!i-D;MMF}AQ1#g5=&eCf9Nc-_Z^+Wal$vs?W!0A8mkOjE>x z1Tp|^5c8UiJlq7Eelj^Ob{9Kr6SNG*Frh%Sd)OyzvD2zuG>Do=TnMMmo4y;Hi|Xx- zl@`wxz8a72Y`(a!+ktmE(n-8+KcDJM%|WQMT==H(ZkszkC-ruPE;#xf%jC5@5iGT; zzz%&ghvhTvyNvVHDn>mx;C0~?*DYWq%asfjCk#H9;(p7yT^nQ^PC@Ez4E4iaZt#v|2#sCCD-RaVT=j0OYUgapy&nYvE@?+HrhPOR1vJiDrR zZ{kaELEVcR+8+7NXwlX{y0aeyA;ex|&my`xVU`AVcO|&TRvJ47iOZu!E@MiLW$}o$ z^#$n2jr{TYhB)l(Mr`;jzn-o6_?FDgasR`x^~Kq$_N|4e7=-M46Fjj$b-4_W=vis?0O({K{iq5!{<5G0zVc zixK1Mk8aTA4U#$OLtp1UQ^ewKs?xk>7w|<^f+*5v)8Mi}C00lBcQ@vbcXAaJ=-gBh zqGI<$R4^)pYs#^7H@4@4g5pA?!u|L)aCW5UV0Afduln`#PZc3FWS}3UCy#aQBHZ=A z6uCMMowiNS3`s`DzvNisb5H-&RoRRf^+?O3I>iO3Th|(D#I~5|?ES2l`idT}y?J@pJx*3e z>6hx;!v_bTZ<=lK*Dg4utevgPgaz7YO-HVsYUEs0;Y(>?P9F1Vi6#I`UwayreZG|F zODPe+TMf*V+e}Esx@_dWa>r6qmGnAUA8R_gjR>aC|Gf(Sb5=4>uSVv@^77yF%H?VS z18!OsDHr9cI}SDhZOJ34-pE|QNh!_5?UocMQIbl_NT>O{+G=_xY(V%#=Q{=^%VdDoE)_saUn2o8T-|4H%s?J_gOg2W)vqHN-ue6jcEeIx5vlG~e= zl{dn4fA)mL@6V+#BzMzwJr6cTQ-TAi;>i6n6|{lmz0x2xBzE7P(oXj z4w&~adONy$8Oka^G=+^&kcy~c@lLB}&ohWS^#H4d=I{#^Amo1AgUJn>Jnh3hg`nAqnPLX?R}6V* zZLZsy@7Woh&T$13XTD+p0iC>1;64-wDQJnvv;xvmSX>@*PstDR`r6_sr1A^}(7FFo zlCF%Df}m?wF0Up0K>);o$NGmIA~G)*c;7eP6kj1}03{zCDomvStdYRFqik&FF+uWI zoraIx>gA=j@5bQa@ZQfmIG!Dju;`<7#mgy2$7>`^eF|2e2 zjOhU;=MX;9wiI+`vVy*BoslzH7nE7TLnzZVQD<4+U*bfZ(NW$qKV*cRYh`X=oTF^4 z@`ET73{@z#QAd>r16JZJsy2<+g;#Xu8;;~TLQ?V#6YE!lWRy~V8pgfYh2+PoBtFY) ze1=pFWSn@p%{T-W9=gC2II$YviyOD&Y4s3{Zl`Wj1zEV|Uip%%q)Z>m?WVoitMqa? z+gCvN6Xol?T1Ec5t~$b!nhK(d8Ce&nND?Yb=*3l~1)Q3O2qET}Q%N;WFElc0A+h#O z?M7(M&h%AnQ23r}f+GctR)2>WW=8h&)rBj-+YMX>N~Q>KtzUt<*S^WnSSzP;vsF$x z7ly7>zARR=WP-{jJ#WgdC8gG>Vp*Q%9r%uKf$d+F*LZG~IdF^L`4$lD_1vd?Mvz2Z zg(P9Th~Vq#(Rm-WXpO~eKc9JlE3wBI1KpJ0IAyY5KY8x%HCax~2ZKMm#BBpWKdbh=`SuJKs&gI8m}aAjV6 zMfN&d)+IxiL3V=SjT>3!cF9r5&P(44KyT!8;0K4iM~ANXdrnnj7`-_KK%5JT0@y%7 zW)J}QFaY3*5}9ZS9;ZbTg(GP&0DmcvPrkbS&k|Rn0f4Xe|FbgxpIGPrQszo@R<$8l z*C?=+@Iz31q}jLgH&rn`umTEbHu+Eb-Vp5+TXT5I_zGU>2Zg1u`yV3P-1FT3nlOZ44 z65h!BRz^`!K6DI>Y%Bj-p!5<$z3ttOEenq>k-5d`+kQzfq%S_H#?`ky9>wP@LJXr5 zS^DFqI*VAN=}}%8v5a${Tn_Q?tVXY_k83!WsbDCjKDU@<9BZYS65rSt#*O z`Pj{1AoddBY&yS?)Ae+wy#Dc6p9rSGeX|%K!Tt-}xwrU*MlIRM3es|;!PATk=41Nf zte??wp#)>UO9uE4m4F8HiQGI7HR_1{yMc5&*p?8_LyH;i zN%LimJdTV%EX}jSg8Wkdb-`u5i23bEuUj$bdl1?r6fS8%nbKC#;;5+>#0y#Zyv0LYtg!X3#mYaN_zQ z;Y8aAeR6b+Q})63xSeUxLn4kxM_hoEdO$&4+h^fB#CGRdb&5vG@Jh9PgzdNNffn6T z9vldLCmZ}xfrpQ`h6KN9Qb;xjmO=gTWX&ickHq!FMjIyC*^;UvO$FA;dL zoz>%WwdComG)4QQ<%6R;DN~XhVn^d=)pwHOMoitMgN>>BLz#?440EtqyAQ3#gRKF1 z4HS{m7=_){jkpN@pPZ&~?NOD2k+G0%qm+2a!>1NI7-*<+qzMX7(q%X@>eFIpAp=Ia{cW!FQ7&nh9iGL*sNcYoO}FryAU z>W%__vIk|q7CefS%TGDKrL!IqypJGv8m{Iw(Gu}uBO8suT-dp8HXBnJm){Q{RsBpu zD+Ydx4jmI+-QQ6ll6bUci;DUUQc(nm<@AGcSX^Q{6>e65_Cq(2RE-OjmgZmZn>Ue-1hZS3Crpa zQO9;`XcqPp4(}w##XtcATVsTd`NL8gLao^i@_DfpszW_YFjw9|ZhPn1J5y02+q1tf zVyCp2U`pcO<_h?oM7Rj>w(?s=i>!#mq9ifyIQ*J`ETbHZ5vn8{O;%fbZtXw|aJY3_3!@QAQn&(c1}+r~ z=i=0YdvbZ@s5(%?eGas1Jbc^({p6`ql$e=YHd*vTP+Ru6-JG`e#L)Ns)j;h8&|e`IZCSUJmRs|2wmcb1oQWQ`wHaOoLchvn<3gW+R)WtK-Xogoz!rp zh?DT40Roz=phV9LvrxHdW+BbWTh{y{B9e{sLelT=EAh9E8|~DP;w`4nW^&AyiAK$P zO+0|J*VB%Rjqm>Gt@{AhwC&i)eF*29eAZrZ?Z@#&e0S~)&B`V>JBA3Bsj~`2@@lSM zR$jtAHRI?9pG!#@bKJ;FiBBl;?S6MpJX;Zr<~E-DLIjEe5iEGbzcV9`rl-In!t%DX zJ&gK&%|>;NgEw0|Y0ap<{<4kAr-p|H{^bhd`67nho$Bvd?TLfD5&j&! zZTc0L{l*}n6LSnd^+&T+Rq7BrGO!m1E$B&{xq;iDKWo)s3~`ADBP zjHc;qYm(lyfJm@yNPv5ZWw%<$ix{Cp^7QfUb^7xZIhkKgl<3iOE8m{KcDK^d@9upr zIrv5R4C{5GD9F^VGPf`xOH3sXGb^IpjY&LKHsa^9n$V=K(-$WaQPB4}#e;=NB9I`> zdvAa4E1+L&p-ldePa;J_V_OPo1+!As*R-_nyu-E&lA%~9vu*v7zslVIV(dfqiTUrSPmwh0nY%h-{Z2mPn?*tI2O??? z)Ges1!FMPq25jBog)WrA$ci0c|=W zMQK_A>_`;KRejhn%glau+67jMbiPzy94l35!u2|*r-6&+(mU7c3=o!u8FLXB;jjCA za@YA2?te*nYMf}aLakMOlNQ1@6X^yk9qI%dvbNa>ZiaW>y_M>=`Fcz|GTUm#M+G#W z9T4~>ok$^-P`E%A_m+0uFHng&*7^jrF9t%Z^r-h?f?Xxaiyz`s_|d^Wc{9@!ih@>@ zN~QeMC<#9Q0BjAd0r9y9MG{4e=jTb#(&`h(%)?jkFqNu|ThLRn`+*^c0-xi+%k3pI z!QH2NiT=;}E?dP{ThMtnG)~HACLs21J6*^CFjg`QfG@P%lKmN_a$mds_A5y~aQ+nr z6_CY>dEx#@6p^?C?MRu+X3i3|nVuAw;qPGfgI#{5W`Vup1XPf7y-73*eYttIfli@Z zbG}8|coC9K0(#4^`b|IKJ|g4800t+{}eIEw{_&?PjWB3EM42c|eV zB=PxFIcl)}0DjkfVZ{_8Di6Jj1uL1o*JFsYXOS%-3mu6|HW34fK~&*RVXhRRc{dq0 zR7NO1fijLv&|sx6%6H|djCm3V*yHpPRoLHn$D-P@Tzq^&?Odo+b?zLjnUmgJRMU@( zA#9!XfFXkmD|*j%pz6>?3gGy>FK>UD4JKEfRqkSZHod%QsBcYfW${`_UCh@&CySLDhTnU1P*`_WIXCdeG`bKh$ zO50i@cQ7fBIO2OJoYMI2`VMyqC0oAyrl@AY*u5dv;1u;)oLd(8gqib_EBTM-N~0%G zN7-|VmTR9(PMMl_GkpDr5`51-_m;Rr_e4SQ(_Vkdf4~1rKfBe$vvCyqs7(Sn!R4>c z_J2r*<&hNXZ0gJB_`%9Qdq(yztPI$CaM*1Jx+WNDFJ)K6i`=HID$ozAvzlcET3wq> zpDt#&bK!{`j-7yDkS(GlM>pz>w6LEq+x9U0kvj4IXdK>Bk)ZQ+`F*vHrfc^z!>&u7 zrzA+vTZrA6l}uRcK)0Ca*6woed3u=D>GXycRT2qiuqoxS*)=G*U-umufrx;7Kp zkIN27obZ8p*Pnfa_GP{NacDY>b#_73W~J_TsejLUJ0LrEG?<-Bs)%_t$KZPWsT~NC zd4}QEC3Nx+gZd$btlqGwPxdou6G^V88Yffg3!e$;#!y_sq&xni@}8MnD@H1JAT>MV z=2_}&!Z6X$Kbm zRzQ75b(1>$weBRa*r&^-9l@)+=j^=kQ`kPe!uWD5HGxa2I|EcR0Z3n*Gs4L8Y2-DB zu&xT9L+JyF2eK;;e?)0;E~-dEU44!MmtIN&s`iV^Od1(EPTN3=^qe08P!BwM7Q!`& zo3&ruX1ME+Oin;r4bxmlwi<6Qu;`H zRkFmJz5@*f(zUZWOMjx1D>5Y6!S~TiQ&F6nw`KiX>%oD8`L^1y)^*6FH{`h+KCF7A zUN6yT=KL=M4=E~hfk&*>A2pT7Z;p<44+)d81@+9H90qr|4=848A|DlJ3R2R6h_him zT6t@gEQvkW$nqW>;()BMewXv$m&k9gzyMj(Ldw&mHzj&!8}h6PQ^p}%e;}wTteMP9 zfxaKG$~@_(E%$$=8IP3|Qd2*rf%`gnx*`h0@0T^ia0S*f2|!^VDg>E43pzDs6`eVl zIbsPwTY{e=Xgk@a89ht*a*z8yf=r}c1Nd!wMQFWBD$h#8Hf=X4-Nj$N3se1J?{F1# z`T{T5no4gTM5&#+d}wd`{`022B+6jzcgXtWh81QTP`GE$dBlGG#SOThV3n@Kpmi=JKGMlDx${p_XWiX|!v8j2%Qi}JJNZpsKm`p%T+n}Ox*)%!c(cC#lyyug zV1w1p{tEuIiLf;v=kXAK2`&WF6WWK^Aj8Ao3$VOmj$2TLzP)u_KZ+SBlON`HeNJvR z6p(>6BLwo}8^)eLzKXT_pjD=ujmyq7t+Ml$LS25mG($c&!vHc%y;Mlb*P%g!Iy3+0 z1ggerA%?)+QvVSmjseYy`}jq6)RZ`MZm`Y+4ST#M@SU8e`VDBT2GUE=-MS=77fJ+T zeM+`s%V1ujsv&nDc>Lw`(TK}%TKi`Cp>>C#I68p3^>0O;nzVz5uFD6y8~TEh;PYiE zl<^v-U)|31G2irI8ryR`TU;^+DZg=y6^5?WAx(JjXTM3>3u1S`$-ZB;C^}PJ{rs{{ zQny(rY||PES{K?$$lrHo8JM;>-?_NHS9pUTz&AM9jZ?Sk)Bqi$df-v?&3*#ekLU)K zq`5&kN^;l8>oXkH6v{@T{6A(gXzhL^4Bo%x_b+*#2CASJEsw?cP~PU45&#T=ylaPl zyJfPnQ$CHYjV<%EP^&U*CyV5Wh%v{bh2_&+-*-m!;2N=&bzeJO=F*f6=A_^L(67y# zwY+_zy}qA`T0T{`zpH~YQ(qjyQzdC4KHQN6k)4CcQygR-wf<}~+)y8sR$iZu*4zYy zwNqJuP36ouI@@i85P~yRNZ{uEKecry%uUYA4VOg?r zT{x4Wp60*=Y@A8fI;_zre%K|}M)Og~{x&+!>rmZxBS&PH=)c&dQk-t7hOUj_F=JNSG-+FEOK*Oz-ZehEF{VU1L zKukcQ4(!6wz^!gLM)dD|(QU%=Wq?Gs?zPp66;2XB-XUbdl?e8YkjRG$mP&Qw<06zN z<`l~F4gU6H5^Apn(G6c&e5&y~UzjX7G7;-spW69;(P^Q^{LsOgzUu8E8sNk(apiM! z{oP$PQ9fQ#0v|+Lxb4i+eSO;`M+8|Lp{juL`-ip>4L6q4K@UBG(Lvy`B-NdRAAbv$ zrjTWnjUgWx2c=YbpzQko^{vUoYra-pH=WQ(_iqg!vbkcTO5||?6JrLa7ui;+l1@p( z$w4t!Z~TYy)I!AupaID8GIBiacHHL}8j^Gn?Xt)AxyQKzf#z{Yk==XG9!VA2$g z?wP4osG8el$9oZ05rQFPA+ABb)+!%&N>MV@i2Zb{NBUJAd|tye<>mdDD!`i-YH!tg z-T&rBbpJefg4Z*E*1FA-81U|b5fEJP3I^a3LMnzXk33|$fh^LHQXuQ5p~&N{pInd! zs<8hZcKz{xhsiHl*dpi!FVuq^B{=8>V(5{s$fy<2D}i>SgQD8=rG8!&)#&O$)Gu%g zuqIkiYYZm0FEi~k;Qi%cXdmJIAH+g4u%UgasfEh%t6C8eHG3G2vGAHG3VY|A2#iN1 zpoKJ#X@0G)WBKL68Q*+&frY#7$EgOo{L(;zL&=6KHrS-2^;-!w`!RB1t9L`{Q{U+F zJrOfcHAh4~)YjU-i7meM$@*vxAj=yaSWf|k=@2QAPhg(Aj;}npsL(@}V}D|zb|f~Y zb4F>L1r*Zhg$86X(1CR%NZj2OBB^*7<+$khex3gK-oVz*gC&l=SB1LEEz_REi+ItYSbjLRa70^gYa8c<) zb*dZls%OHJbHzfcE-Y6dIZU5bnii@)7f5v@cgwE%WKjXDBWnAc{uHnk`w@Z{Mk8;0 z7h^Od6R6W_R61OpdY=pRN(}dbSs({Z1yZ%8)lb6(sd^5MA}zNc9W8bsc#$;nvRg5o z_r02tg^7rICur*x&*S|nUNhtTR*yB?r~w(aWJ1j48FYK=J&AF4VyeF=_?WE6{YpAB zrHM)tk{CrJuYKnOv;3|=c3@fGjiCK2-hSILaI4v|pxAA}37@D=t^cRDA;g+JLiBfx z|HwdD#&msYqG*bPV~IRR|2R+cP1xDiIyG`G9OPU(VhG>$;@4hpVEvEnUdgVRI~RU6 zpAiZVOR9=(>X9oCE?h{1y_ql=nyITEDlw?D1M6N+3?lfqG;h9DLodD-DsUil*AkI$ z7`cv`z4wG-WTA9a-tR-6p61BQN;TU-Eg(M@rL;>OC9>ro;Kqr1NTHg)XFWGMu&tF7;X z)yIijMf=`%_H~Bh98-ntfiw3l>k``{gVNj3%t#Jq*bLdLaUJv!n z{#%v`zy4zlU}xa_fUbqD%jd9>=dd~Ny7+CPJ-GFc7KhY*p9fReXWBtGAvahjdTKGo z#rl5a0>%)$6Hm7$?rrGG>%CoQ^z=en(+@A5kW9#8NGxm*coTF*+(QH zX-Y$c{`6{QRw(qD#9G)Yp5%&ET5BE4d%oAI?bOcBNjZU{2T=KBNv9VV9xzFk*K$`( z`Q%9pSQqo{4|9-ofBimxm73Y|S)E=W0)8lj3ij{*n3izQ?+V|l!V+f>wx7y}Q}9nE z>V~_)q=#r)X)2#gSYP})tnhpJl!M)!YFkN$@J81R@s~{Kbai#4G0mgNpJ^7XF`PqK zNQ@s~Js`DnpE)G~5AGil8ioa0PVg!uP2{hRLlT+(shCG!Cc-|bxzFUJrrOGB~@=0g46A0KlSk&wFnO`&)ao=os(AcLiSysAtE z1eT)RG}=02tK!1RK(H4eAq3*$nJdc81h!A-In3?TI}q$xJ*Om^4|V|8ZOr*}n@4>W z2dQ59DmnHr(T#uPE(hVLz7Pk%~m7|V6!-M)7&0%mCDj)4fD>_tlphs zeNTp-wJC*Q7IFTwiZ_TfEc1owA{l4YIXulkO?Bb^dOp}_x2u#s2sc9&T(<;~fDc7) zxRdSI;q6YVzk~{dbn;dp^pVGQhZ4((1ji?^~# z_`5sQ?0U~Y+-mW;+593%751hfXPGY$F9@A5I$w8FmTjtbEex0QK-mk{fCxbPSFFD^ zE2Ukp+hz}ZR)%j;U|j`wdgdmHt!}Us4q>znkF0FNWH$*A-S^R}o5vee=(D{Hw69DD zKIwl_b1jhgrz!8CXVN`6ghz6lCTN~9L0jh$RtY%?`gd-G&+L?_gi7e)1e(gblYSWG z$}jUJF|#78bOw_CVXKxN?^d(&Kz>fgjtj2CiUVUBN|EISuL2{7m|+8DysU z)lT+R{$>f#@|zgU5>$O_I7p(5<6BNoCz}B0imBYtd;F=@Nn`xPU3S1t3PBS~pn=Nc^A5)G1`xkP@Q=?(qQF6__94E)UNcN82*u43|q~ z=CA-ST|Wl*ke=UC!fLj^y37rG|1;D$5Tw4d5Dw4Td2QpTB!R8OQKfSo$!qQ+z>9@F zW-9)<#ajY_@^3qGt8}1yle;$Tbg{Z}d#ICrSbnz9x`(EvQAjIpLTOl!PC>1kr1_8s z>_0+>uLuGS`$kF}W#DAESg$+obI~(YfG?M+-X_aI^D}9j_8or*%!nUO-JzXuC~n;W zg(`Ym$!yS-G>VJy_TWz3vnKRg1Hd}YdTpzLB*rwQT^$ko8fTdmC=wvAd4kfX&HB&# z!rFCfdLedKqzh3xsa?)+u3GsB$!nBiz{3cI&u8n6$PFCJyGCt2&vbLfysyF^X*FP>86B-lXR_RPit2X#S|M$5pt=zZlx}Tw&M_qS+qW;;aHbL~g zPHs0%$+OElP#U*zI2m5nvFI0O#s^6cA;f|%wvfY4Fu&steCbh0WYeO58^^xcDt5${S!iRsoh27?SvpU`pwyLasYnWo z*M_Vvzdq|_F?5sTklKi1Qvk&k08U0c*|UKrq!hjU6Z=0B{6+t{y0yHjv}!vHU%-{k zji}fpN)4bMA7cAkDp^%tH4m{Sq||8-m8~@G%&j#bQ~3uiL3xg9=2JNd<-wTPzlmtZ z@O0i!CM#_$nql-LAMoX_{c_1(Wnnu*fr8C2iL2`gSRR zO9u(It$6rHGr8}EmP6c=Kt#bm$Wl@Qg|4kKXJ>bb@L6}v#Mg;^1Z(BVPT&LYnDion{e=C59x z;~Ob);%}$^1R{R?nV+4WhxTs94ISMxh_9e_2@!z zV-%M2^DdFT&R$HIz;;-5#!mZAVImbzpaOo>X;EwH16y@_l(DzlYvnA>XZ=73ccq{k z^t%U%BVN1!hPSH0fGj@3Y8qem=uP$r0P`fcW7<@p1~t!|%b?77f(}ez_|*}#Dz}CpbC4A9Qe~J&i4=5{z_k1n)Z{x01BTUYt?HcbQjMr>Gm%xT~D#c1s7wy`7Alg(@ zcc>4tbC`m5Wvl&cn-bjg$B)wyv3%RJlA z2-oltcTg|p;@Olvp`G{ebxZYC(z@;_@qx0_0Fps+Vor{+E#N>&c?*1^&W(BkSVJW| z?(!b`lup0C%`AyZ8N)y?cwq8QsQ#rzv;w-!~I6rL7 z{5WnNns-d8HV`mu9kr>Jy6`0SGXq1W<~cwU*{m`20@w=+qxb}p!sPvlyW24ye3lBeuh3et7dnn5I)-Qs#{eDzsDg*#|`*`=?nD`jS$Fx=o*QFGEwYG5R zVQ-IYWwFNcv3Fd2M4`pFb}0mN9)?C!&{X8UgE@NfLr@t?&cgg1JDlh+6cphve4HKQ z>kakOESJ;PHurHR_XGQyTj%Q%32xfi_C0$d!Kf4@hzn)5H@ire1DZ>aFu~S3iUFA_Nj16!L-O{e~|G zA*d9@h!U}P(&7m&i*nFq$3{9BATqpDti8LzfQ;Z~?ZV@!LCy0SQi522z`X#nd0kDV!MbI0xg7Y4yuYcfJ18bzwpGgnifT_zD!W%xgOrxOCSt(7 z0Bmp!X761^u(1tZxJ6+$Jsdt8a+}A`aNLY7IS3baY{afhfRUYM%hQvo zm<8C7)R5}9U*GjYT-&hky?yawrT;F?V(h>CgDW!s!1gcy(2vYNeEXNAApV!6*hD5N z^!_C&{!f_%+N22qz1oh5%e%5|vOYo9|s&lsx zz~~1fT#BP#@dKh(FR#nTS)=pOaL)a~{Pc#qty3dkMciIJ>*G4+fACf7)oUrHc%^PC ziM=>`Em>QV!j&$RtU5BK@+O3$*U|x`s1)5+o<=twp0=Wr_|tov2Jf)M_&dd1iFKNG zjaK`FPHEQiyI4$QBdwwUaO!MWvB8{Y-;edr^|;I^+pyPNHf7r-&Iu4&BC7N{Bn&*# zeb2*Fkt|uA`d>p{HDwSdYZRukHFjY*OF6|mU?LtKO~6<7FWuy+&cE?eIjA`}|75jC zQ3IoXPdcMX*KlgHVSC&1DI435b>7_RYo&TO-56fjuU6j5TQhK3M@$*bRZmjoY`4yn zuh}~dWE#t8h|yZg*LGm$u!ua5mg;p)VnCuO4DnNgn(1_)OOai4bxhI1ujE7aoA&MD zX$qL+2vdJsck%SiP}hq(slVx5^9mvw$bHgn7y`1t&09Ue-cet8wiGIAuvkKTre7)S zSXXrFz;XGzc{=kt@^+aJ$#GrdwECFgAZv2I`bNe(|PZVl;ccpCmx4(zCMd z`W}c7Zf(pq9iW+){m-El$f4b>=8-kYvwmw-Gc~YD!aD`rQQzRB4!-W@Df^kghAtEm z62p33wSG9I4<U=pzy-egXBNJs2tU=pHwHsT~cD>b?IoW*}QDq zajflUxq+SjxDo~>yS#>ZIN}d(CMWop9%ku8~i&H7t! z|As6}{x!4n;J<>zEW52V3>^5uek{Od5>{|G(}fXyR-P{p__p~a@XTPo*d_ag)7W)W zBdgJY&W1DGe?y?(D0zZ3Fm*iJ*n4)NeY?&Q6bqX|r!Dk}Cj2>TZ*4Wm^a=k1p`ado zhD#cX@PorvOeowow8N#FlY}ya7nNn6n!VTXe+49bxBU%+)e5ds?dK3 zhU^HCFE#E0T;67{j1ZhuUHoy@j+|-WtPZ#dfmbMI85IJM?)D6zx5W6A9HQ~t5#?n0 z6%6BFPqUXp4dA~S`2SmPhySmbmSMjfllF{KB+oO|-qt9wv&{mY5UtdcDfOrUS9^(^au&gW_J5WsQLFs5qKB(+d zNrus=DIT6r(2sl~4Ij%7?VqM*3q&^glcWaj1+yhGVr7 z3Zu>!j5$g^18*nF92M@1Bm=90Xuhb0h(6j;;a3U+3tLt5K1BV4$Z>B8-En_+t{ojLgKARga)3?*TX9G`l?SY?nCAA2UFz^a8ZkNQ~6C8<*p>sHSPE5BnBr2mK#FMO3jtV;o7C^*2jCvz0iovgc)FJ_y~TTRfPTZ zp;=|v;WxykAQR&qB^Y3@0y%U z!vs+`1ks`xA4R;nMaf4AvxfIdv=jCjFEP&#N&TCi?+URsY-bQNz%Gr zpiTueWe4CzoJIg%7%71p9aB2R2m_|N>AI7ff2v>q4_l* zYc>v8{nBmtmKPdOC!hkBE4Io-mTssg9CG(3TN^9ylitvvIN?4m6wsH7*0BU|Ik zD#HnUQi-S_O|~@I)6*LrQ;!nq>vAz>M)piqykt4pFj$7iq(pH%mN|(3+q)9(avcqC zZX4E5pB9bSc5g^0(g0|9&>Yesv2&TNq~VA@Eq<)%Nvwf(Hf;NqY}`c7XfHF|Z)a;s z#P62Jei|e>EJLNHzua5<=BP>>x2w_~i;x*R{L1ltUEu1~j%t;>Ucb+%@^0UFH2-iO z=8EJ&d2wwq-BN!p$p3ZCvFH{H$+EX2M-qk1K@XoKdC>92DxO_s> zeV0B1m>_@KWW3gyEAQWtOsqhk-s3LwMUX44?vqxu#ySPT$7!?j<3iQ)nEf^*o$C*8 zI#C2Y9cHJQj?T2z6b-M995I6j65Q?RerNOL1seE?{x-1B?%Y{d85NRtJ+r4es6U*2 zNU_d0?aV=yzpY=4>-X>Iw(Uj5-qTh(v+CG8s>}nYFjhLbcA|f$nl2w+dDxv6=dT)|R`u4zD5u~UTRIn{k}1hIGW9TdS*zLBavoWhp^c`PbM?8O7rR#1 zK=hYzG(ZfSCooG@w@B8bppp z%hpwK6749~3%1!hh?vk}G@^n`Z&*GE0k;&>QHI@1XV@sA zGnnOadcIoWuE8%0oTUaAJl#7`Wi;8$(FqE+Y2Zdy1#FY>*d>%o1xNkb%O|?dFf3qe54y)N zwxYm;H_*;lIEFkdc?3OB;8{lH(N#zA$FGP5YF$UK@)7C?OyR)I03IL)H~^l)LhDU z6|!JIM*P!NW9LXY3jC_>1Or=u6a$riA?XDeP?aQoE*A+{mc_LktvO2vYbG-Ky{XZP zCm6wrdCgHP@O^>&koHKQl}i;|k`QU|Tp{C7XGliPK@P);{W#{|tln5BY-D*Yk0J7) zU@8AwS0K<7%AbYgwdG#KIDtp6xq0_jyhX9^6uOfgjo>B%cW0*EP?%1#@$a=e(fMO9 zvc~HuZfyt{0Odo~Ct5V!jWxyK7)=tODMuV0yQww}x%g*tB}qgTY9N33Wr&jG3%3&2 zqlY@QooLYFBR923R8XO`HsEhM;Gly~Fr{WxrNlx*l|o$?g>H`hFu?xJ&-M$$Jll02 z*dL-^IUP;3Q)nyrxNnJJ>6T~;LOFA0^2-ONMmDSn>{L8sQ%y&;Y7Kl<6Lsqb3_flM$vqQ-Fc21YLa z_%H4-A_`CYau`MDf-~$3ZtEG%0*$wNAO*X-ugo&&ynofYZjF6$&h?R>6;VR<~)1{5`3*9QZVuA zO`)3jty~AedZ8(w`pHY3yVNc$4D7NaaMyc;;^Bz`UBG~WsCt*Tgdi%z$BS%$8(}n0 zHA8*AnK#Ts#>Qm+CukjjLn($+bJsXj&i!Y#4ZzK)Q7`tYqtOu>A?X7d*x}~}(YS@~ zaWj&1&j*$!nG~gLSIJ1*<&^u~_}EFDSf|7oUHq!N4&?yRPfF7SWQH3;PsQN{g}tW$ zp+2wWaliOKtoZ|J1w8_rV`yT|RQa8LLAvOjpz!(F7PSPlh#}j69bov|F8B7IP|d}F z{`qJYu}iNlHN&@@cKvC4`0eUS&m~;aJ^dBbZ`f&NA%D>48`41y7t)$VI&p8|YL0Xq zH4D|UOe47NqCf8XA-1s+j$diM&{-Ch(tbAEtprr5<#W_DuTrf9MSVI*tf4LbNo*pn z4dSo}lfvJt7{N$c^VBjCj+3SDae93N8evpT#V>ss6Hj?3V;XyTRQ{z!43C%cS%m1J zy?{5)u!EU$Ki?8d&w@bOWt)f0I~4yp-s8EI+cLj*i6P7555}wr=O%0+3y`9W%KbLw z2wyqPK{b`ZgTb)lgj8m!MtVUTdo7UJA&&h{6+gtk|GvURN-M6d?%k#MWF6Zoq`l^2 zjg?{*;;aAdbSBmkA3+1<4}Lt8`w1YqXn?2;2x~4(I_lkOaaLRg)bTU{MK9Efkq`0& zN3X$!)KzpI1L9QtH&&%P_`tZ&3hJ*PVlcruWG=YHx1NmK2*Bbym0yOodm3Bbfr$7I zEn~j13ZSpJh{UVvjh#@EFWvjEsZh*9+dARZpSd}igtb$=ZN&rAmBdmgMKwCQiUB{c z@GY=S)BAI*eoZCHQrD54ew%{PwNQ$gA(DT6I-^P02;pZ$ zHQ0tj@EZj{=&3)FGf4VF7~_~{H#&#og&l-yPMv3LbP z1vXKr+~b7gv;65-iDY295O+HNu0v==bY8JLD+zbY$Nm1B_Z_eC;sP5C%HR2*S(gpR z*I{}u80P7?R~AJjlrC}z`SA!9c|*iJr@kc7fVCAyQ`^J@%SM+i;kTnI32MK^3^C`o z6KlT>E@_d&pIO>7Qb#j5$K33S7^`>~`9`1U{pm!XBL*MN4H#-roeMnh(p!Gx;aKq& zd`INbd~EaYx;e4k#^$Jt5fl6fwdG<$`xf_-fi*2?*Fy}oWMGnh^Qkfp>fF^|lo6Q9&PR>#`eUb#2d=LplZ9GF1hThwi=j4w_qx)R_d6>?lSf5* zm%dLaR9PZKJw+oQ8?+&x??}AJRGu;j#YsJ*MF`G}9$M~sjI;-Qr6>r&;DiNIuAH4d zI|RS@yyJU*Bc%&n3q=u?lo9k|(O!k))EU%}wmzv^%sFCU-D9EN+=cEU8pHRK(cSKZ zA};anAhI+m8FzKyl(EQjqSqe2XpA=V3D}b5LcQUL&Nmnv-B12=JkuC+Nff=VaH8&q zbP!74c zK5sx7bLo5^FY8Jcb6LwnFN8g~BEk%du_hB6jauiywmrxOQ^r`IP!RzKd6Eij|8rfl zry3MzkjDpv(P>RKGR^z{(@oRVvNVsEUZTkbp?IS)Mm|5Fl<>5_cUfflRcxFdjVRp@xRx0Cqd9POn*o_j~=(xA@T|p0wQa^QOf? z63ktk=BZ$ELg&elpK}7XMJmKOak3TN!ItaYy0vP=IlO@)lFFuE&e1+U*vqenO(#Jb6h%?tOybOUG}0e639L)ySWr2(flo9 z-lE@J_L*HNWS?OHWai*6QN8|8nEoXh6vRvaqT`Bnk6|Dba{e%UB`YaeG0b|}E(#w* z2FI58O?9K+2a3Rj)gnVbK&ALmUS|VEjg~EF^xV(M_x`fDq#9>ES6k3K+M9aC18wTq zO>hw^ADvd``Abdrm(REja^tV=|4e0h9$TQfUO6_u{|{ zw{{=23V-T*dT-NZu=j!tewl!N=Ln`tIXdw_XZRR*S>QaR#6v?nzyY{-2^SaN#>WZU ztjxi~n;gV;q~Zs>@(YlOM(7Z1?7|d{AJgT3MPPKzx+5FP{-zfI+|t{VQ`KC4MiE$7 zurC7gF#qc`e(X!ukV^(U+&-Bo76P}LQ`p|51&(EKrYj5oym(H3l-~3u4oIS6I|1SK zLff%3($!t9=x#qgJ}3_HqM^czm0}~$x)z#z$vb~_`^?ttNI6`2T8+Y*CDg1CfWm!D z?`Z@SqIM6NKPI8ydrC{ty_D8A$??_2@Hg zLF)IyJ9agY{iDe7il=^?3$l)KJ-yRu7p6QH@0}`S?0ia$^096Zx+kTV$?izhmpJ5$ z(qj4YV}^L>k`d=EG<=nXQQ zg5qVKk4W7k+*G!X1k967<4X$lY5z(578O3Ee1!%kEzgqY>r_1XTH>@%3MNIVW$wMJ z^;Vi7T$P@tk6-&hTXMv=w%n;OE^>3W6k%&~+V3P#r&pu&soN~Aqe&vnOex<{fpLpV zhI~G*ifeSj^^0z?;~S?@SuXK>?9=bKus)=&NVwLx2S4;l+q57vj0R z&_$C**2_gl_`s0>$zpk3JeHCv59oiNkMpW8skny((cNf^-ASI#CNp^-kP^AVk z770nR`&bZoZ)Gr5(ijWFs@&ZV$NOZ`(W`Q-JUTI*6x8bJr29!1wO`r67b~Tq@<<;G zjjYH=m0*283`gltwDMrMvG3B?wq{oeP9Be-9lD`b4aI{e%&|lr$2SIYq9y_yW9;+A z@Z7011))9>N+LW&{}QX(Xf=;$6Na)R>b8&~NcX?!@MbJnMDf}rG-&%y(oIauE3|d8 zF>&-D>(tiogj=8CpJ(q{H3!XIw{GM|n9ch%!sLGT=cUn*kpW>`o1RDO-@l+<1#f^M z&xigj6cW6Qc3ntuEETl_uyqm*P@wdgoiC@8X*ehM#Q02|X$gDxt_vmCAHny8=uhkx zk4futIRZ(+M8qXvjWn|h@E)SAuP;@1F=5bj@5BLDC=!s<7;kKO0ecx0j3({+N$PMt z_f5@*x3&75rXWW+yjERMf1#gs;9Fg^_61*Mbr3N=mf3t1_~a2!MzFu|*_ z?oa57KjIc9j&Tv7&{9Q=%55&=>J&>Ve=A&N-SNIR30pviKsp6(le4qOiD2Um*?T0ZkAtIc1R^7r;+v{(&c;$wQpBp0SA zLI&9VS2nZC;RdNc)0cWt*G9UqdjE=_9b^?KoFQ=pmS>xDeeK2iYjnx}bsqbq{|-2m z@gd%e=cA0pKLU$W{Jab`wNF!in6%<=?z!0hnwkC7j^OYb7#m*>UoV)JY8!n6Cb~hq z(~nLp{=%~UmgiP{aqpq(C^nSJ#D)y&pK}(ss9b+v3;I`5tvlrghVy? zZt(9A+M01m>s4!Ff7k&79fRS|>hYY(ZNCmw;4fQg@sBV+cB)n}gJDyqP=O}})87;s z)MTBkb-^cDI4NrmgPWK<+&U=xpP*rEe8byCR1;3Zoq6EE_1O(C19F5dL=zvJr?3M( zumi~mR2>a)P}HiMJL@%aP_3W4Ip@Y+uyo0B99Rnq0Vrg@KcPa5i7z%omk2- zr2cF|!<``La*UjBJEzA>2a~d-?LE&VR*pyh^wWC>Zhvnq-))Ri=-8JGA8|qsPztp7 zPF-LEyM-HBWRox9q!|N-SS_KqkGv|=82(euXOtc@nKmeiAP$>#zb*NYFMf+VJg879 zdOjts=<^os&*qOXvA=UJNhpe+#s3lQ_`gB)+9#zccg(dyr6Y}Y8uA(UIN^iy7gKdJ zb50-?Fl^vbCWaG;3X%J3xt(&ULqm0LM>gXe-kfZHW&} zpLW#}4^t)u8E3|Y`EBNIG{gd3^!@+D?|Yl)g>lfKU`#FewSNZ7gP9hALFqjfAD3l6HGf6x$3(q8b z|Lx&*=DeLPDP@ofrU4$WiO@5cQJ@VD0Fa{ zGQ_X--ZKZp!5?8pDH*e5EwDflEVjjfQB!`N8?5=d@b%+SPWoN>)G?g>io~`!v{37B z3p={jh8n-arY`L1Vux}j#5lnzEdAIhN0W2WG{4E>3o#I^4PD_{3=7y_%tR-33G- z42fDOY`tCo^DC0y(ODwEVj#6GJlTj(oZv_ZiwUE^o9?rOvhIC;fRdVU9j|*);C^LO z_)0$*&mlaY>o-*8T*>=43i1a4&1J0ng)_njzF3DXGu;l=S9wila zhuXrfl!NGIKO>1uBF#Ns62xrq!7KNK+_aA<_lC!69dj<=SB#Ee%noAI31}-EkKCUz z{VkQHDrj)k8irN=q^;Pu=>VHh?n&3qB>uq!_NA7~ze2k-;Li0o22wE(LjQifqvZki zV`eYil69*8qP2$YM?gmLl7g0!o<=mUwKa3nMh~@lOzzTJG8V57$U3KmL=dVec`vIK zh6Iq$ucQA^yfaOYT9u>xToOTWXKn#*uo4_)Qf83t;MSYm$CftQW}6Ytfr%6 z82T>bi4Wj=$`6;^G?`BLns8yOi0c8jU5i`qRy*{b?l~D^mkwCuPS?Q(BZo9(lX(A; zTklRMgK4hli7Np+bIDvmpEM9TWCe;3WhN{;%4Eykc0dUfgAAykqJ$({d0e zn|@mA#N>Jf-D@&4s6X+tM}tgISqqwu3cO42DZUs{VHA)K&$9|^Fu7b1;8p8Tyf2Y- zo*CM-rhd@ps2nQ(Bej{EWahTi!+D>VMD>@XKhy4nTl4+pTPZ8wgDqRwg1{Sn$MRQS!@(t5h%byBAICpC77Dsfk!4pxJOene? z|Cqsh?P4q+XtW#EeJO2gN4bBOsIb-_LzQ1Veos(_ViZ~;>L7dX(lw9WAKOL*y20#{yJ5>>67^4Ssuna7O}nB>7RABT|i_>p1 z>wDK_a8tzz6P&F4%RorW3e(!*^@|gbp}pTJUHSEE66nBUx((Z?f$a_Bev?$X2H!v_ zd72@yKjV*`JRMl{y2ieY9#56&4)8?f8?+4hES4CHCw95^V|Q#_k)WfcwR50ydV+nQIfNn{KYFMF4Ng%{Tfr9)CxHMG>!{9*$4$3aWkb||baL`# zmsv=h$nUH+HB2kJl;ZF`DGJFz=}*%=E9TKZHGkG#zc>m#xbw4ha7Z+V`^y^$F#tF0 zoE%t$UOvim(a_kWk+;>c)02j#@)bW6e5XKqpOg83R^rNmi$zAa<^Q@T!`;WkniAk| z@afWO^CN+XFUObKHjgDF+ip|$DTJQnf{IiO?uk<&DNLP` z&h``zs16^K7XoZD%Jz-zMs;^V%+Hg~i*mSaRj$oK$!5Q^ZBP6=n?#hRZ$_BZTO^-{ za^8(MrI=j}M>jZjo%|j#=BUy&Sfhf}mgz&gYs*Fc2E-9zh{5r*KGV;{8eXqFB#FR1 zw{AobNuu(nRZIC@>EB=Z$%8n%fG(k*I&tw(r;!g1ud)wPex;PWCS=?{8ySu=%{@HV zoN6vxHn|qf$}kvuptw)XCc*2{!p|Bt2Myf3nAf(Y(O5P0)4srGCVk{V$dX6->NsE8 zs33b}#2OaIA%+h1E|~XCi}d4?=KxY9!zNIE!P%Ua;dQC;H``kP@Ef*o%60ZXtj$*R zO8h$HQ%z$i+Iq5I1m$eZ!j9~;wovumYW^6nfQ0)4z0adme%s+zI)no=(93_w`JTn*V`d08}={5MbeMDZ{Y~Bef-j!YnDVn$;!OL2)$e0 zA@AHWiYiq*O2<=yBVk~b>*rf4EkiS}KOg;r`~>TRH|ZMQ+G{lL@_Eaj?Pg!0w53H; zF37F&IFt@+=d~Ft z;lB*o)d2YFqt^0RR?@p1-Q?17h0Vc@-iUF>M_kdTY-Q?!Fl58AQ zyVC96vf83ly}?5pIggeTcWmeK?QFP_>Z0fKr>~-ZHEDPMM*G*^!5$sExZ>Wha#;z| z-YkE|zYWaXD60@H&sO_cyPnn5acc40V%NR?;TMKIa@%8?bPj-*(AZuws!5 zfDe*Sr&=rR-;j9unNs6zYTwn>N-&zc*XDZUjuw0$)pDcQl+U=i(d5v?+}zN5jDGqX zN%2}8;TNj_n-KTyXE;BvV1u#2MTeyNmya(tCmJSOvJN8IaE7m5CsQ|Xz0kYragU=s zX#RPIJj^m%Bm~{R@}A#)7V7dZbx=XhZ)4JouR5POAJQGP50-l+P_s}H)XPzTT6*55 zU-qf@(8WmEY)e43Wi|G&cL{SN=U(3Q^QG1DMd=+3sX&qY zeniX4TbPf|>nWtlYLp~&YB9l@F;V!-xB=1P<>?H{v1!*rmBipuJ)byzYzurFqbtLj zvs~9gN;k3?k8j}a*`n{{M(JFos#&U8Depp;&}N37C$FDrZ>ANLJgDMIJ5Gf0R<%og zo?i^J+&?d8Bdp;CwDoo=b*Bxzugr$vR+OTD_!gdl7VA(R8WV+`=f5Q&v zbv68nd$bmBdgsa2pZ?(_rK5eaZ(7663Mh1SR{AJmvUf$uO=9jp=ph0o9(_i7==dzN z`*!%O^ z8cADnCa@`=)s2nfrNbUQ4-W$EWV^(yE%qm(ieI zhC&fKk@Es*8`0=AjV&#K1?HlS4T&te{}*U)rtHV^pVfK&Q;tVoBa5ZZ^cM`kG@3d* zN|n|pn43Z$^UmLkJCG`aq2LWndBXHe(Q03@$;>^Gv&X-U+DXp&WT`7Tb<*EK|S%1-)W z;EOP)G~2e0#8bbL;4Co|RUqjT5$n7yGNNUf}UB{HzoCdk-%&nU$FY>arSqe=-sZrDBQ<|L>hSP89 z+MCnz@!=*AXXg51Mx;>=s<%VGKh-eOc=%!9(z{&+psn@nu7#VT0B~iPA8V zFG&`OW+^0ctsYBC4QGP{`7VV1q(bg_kRl%KxYE*~e2Cnk5W_7}Ds&$<5iyof%7AAo z%MaI=#=LSpVYWhp?Ns=^GAl`O0a;SzB{>dYm-aM061054x~U*xKrB=c;QMErGLYpm z8!}p)1TRSn3M)c8U*~;|3DAXzsT~CrSCIGx;RHJU(^o^`WSTl;tO}K{ueE6&v7s&%KJzK8PEMXEMoD zWCTL{!`^Bm86(8xo{Au-K5TU5Z$*QM${>XZJq)eeKnvlr23F&2pM9_+ihw0?YSMhg z7h31cMAdDuczR>JJl+vwAvxc=`z;lCVnR*_cgzK9mv`hmWiEDDqYe5VyVX-HJMf&2%zz(krfy_f7-hOvCJ zq`(HFjF2i?AKAHwDUE*#b7CMP^LR7Q?q$=yI4B@6^F(&<^@6^eeIXZRbsi zQFwiovRgtlTbLx=Y3lMe{DSSb=x-iFmeHiAkQ0$lzjk^JUI79Cl9Bn1F`#Kwef-je z7Yp-AR5d*>6ZCO61uSd6JiKd>g=wJ4@@;3IhkOFJAE^G_)rO6Hgy7Y<_&vJc4tB{k zuU4w!;=V?(FRB@qlb&2p;kx2n zYiFQsLDO_qKXg6>%pmW6#g^mzjZ4tddjgx>q>?XOaH-9_;bVkQ3d!vGbXEQtxjV|- zPe^v$0(*M+-G@Auh>Tc->Nxq;{`WZz9oFtZ+GDiy>zWCUjr~t&r?$t4Zu{tU1@6Iy zQkAO6Z$~Ne9FDtJWHS?ZnTz7DB4_44jKu%RicB&pvaCW1PjO7^^ZlfwrsPQ&{B^Xs z_R83pJ}Sr@_8naOZ%YyP&!p#1IAr%uctO85?|4pqAw6bpzSP-CM^Nf$32m=tx%d#Z zDe=`W_=nU&H#oY4%$iT`;?)dKlf}}dSfmFnD_n)C?l~T!z>ODXu=7v??WFrQI#pRv zi#~#1;44Kqu0fsB{!^1ILeo~jD7n}Z8EVrLL9i@3!skRY6v0HdImZx6QSHJ+vmwJQ zO@-HaXco6kdx6ns1$6o%SbL@c>^iu%-t-%sWDyoAj^^l<-iQcM%4)o`r<09iJxw$# zKR@*qa)&b|-8ly2F`vVbJNnYNZRF6)-WdkZ{J^A#T|L=AUSi48Ku@(SQ38mcWzp%A zlK1ibm7~&!(V^MUzWlOZVfDw|8D+s zdsYa$+HAeX>EIO<|L9%wSTc<@!_H{fUo59w@l zZmFW4dh{#>Y1)*)Z7HZcWM;>Xg*C#SO6f}qWv+7Ljz(IYo#i+-KaB^Oh}}sF=kZ6c zD&-~X#|(xo0Q|9XgK#s;vCE?BP$zWiB&jms*#2YVIpWBMb13B+uYvmYA-F{k7T4)c z7_2=}6o#QQvkIjBkye2mCohL3NwPqIqglZTY| zX~>!5EwuJw(}p_J_uX0`OsvuS}kgTDC(52@`Yg7c%U-W zdo0c`eUL<2%30q#2^wv&g^=F}eI1d|#ABI={V$1NME?GqIY*!<;4`pD7vfOPUi*>a z2WV8ALFi|0Knm2+G@>WKJ2!lZxrb|V%gni_s~>jPSJHh+RWPed9ybdpsD~m`3qoPm1h0vMMgLd8WxqQ8cfF@aDccwI)9JRrMJiZ zB;itQVCqVti;(tyJJtRHu_-)7s0F`s=f|)Lqxpw!6|G1%m`LI2 zpONUN4+q2vk@0T;Dc@3r7AB%v-kFiP(+yL1Zy<_5A=ES_M}Cv{g~vKVHpbl`{sFK* zJ#>x3ofyl&2w&H0{`l|_fLDjBK^dk}TEG|o)^jWLeOaz&S& z@i@%I*k<08JyJ7>i`;QHejOMmKaHKirrqOvn@TtFW%IQuT`gcU53ky9cx&ix3yk&Y zSt%774TbE>A}myj8D!n+M%?`+goy8Z{s$XK;71zX;+1+rfr~P*z12ce{14CijSp#t^R-atwC@k2Bma_U? zduI`D<^j@+Zy{7k0p_&Bz|x;Mh(!(wgyR- z*7^*g-)w=pR&{DrbHZa1f!7?e+tDuT`3~ShEj9DJ;^EeV$Lxh>%PFN_m(%sjY2x9R zM%?^qx+)BI9=%h;0ugJnudv<31TNgsl0tc=-p9R&zt|kz-stVV^?UM}&Rw;}TyYJ< zK8Zk_M39reftz(;CRt!My|-M~5*zGZ-us^IUb4#Nr4V^m<#)dM@<30uvxA=qwjl}s z)H*8||1LvtM}HZ0A4`dIUKg`_-$_0EztO44NJ zP`97<7}PXKP3Xaji>AuNao_S9t`L9ND!Ke(lnsU|XLphEqdt|UarZ)KA;oasXN9n+Y?6oBcY|E${J-uab~ z3rxZpr6z2|wwEB{3kNpJ7SF);urJ17l4<<2PFQPnCS&k@_h^?PS))P2Cxk1i zJ3LZRbr=bWzxyA-+XGs#I%ZF4+dAk_`OxQIqAh3=q?KCrx@9lQx_03fuT0Pn9VJ55 z4nIy63{L7MloE0R5h-4esB|q@5F9*7EZu_0GvPIQq=U91{^iy6^kr!sHcjDwd~rC( zOb6KKY^TsEU1u`vzdX-8*z+g86H{52=1LOv_HNZS>vAY2Com?a*u%-NaHH+%P#{Mp zEz5K|#pK$S=*GgUu}V%5%1z@c1ZYa8fxD{<*Y$1{WJ4bNy^kG_MuVPT4yF%op7%SD z6+n({^z=f!YoJyLJKo^z9^YTqNF+Nim+VFZ%-4;X9r)oD=%wfW=LdY~?XtH?Dv^AT zd_bZUx}7>A0$9u4WUXB#?Hf9!vRmALIQMI^P?0PZY)%YBpU1Gso?8i0U$RSpJi)&mFRWJcvf}BbH85CGOg?IuDQt>@%`(YC7m&)eNTle zf^}YT7vnm9pS#2)D%n0(msIXP+Ak>Cd8L&|7*@y7KvFOSEh>+ZPkzMnn0l?^U-Q0hn2f({0&4aFRq!GMJjLg1~pimSo&<{7sV zd6)U2Wv}@FHIf+ocC4_Knz(5ZU#}mnZUWKG=Nltwi5zbxdiyFqa0Qek4uI{W(8KN` z^*isSi|P^m3E}H`1$RaxVU34TkzwgrB0JHYKu#?#E|^(^K!H_E=C1LX7A}ul&oEqz z=qa6z4}K;Bz*9~IZ@$b~A#}n{BS|%bd548)GJ8jsJxHFW1KZHFW;L=}fap>X2%ac^ zXK9z2?B-!B8af3YzQqi5Dj$M}rMn?bm%rruhkXpX@KqeJ936J+0F!9FCp+co>ch%O zS|1$wk<^nm1c69a2?N>4Q<(-_8O!U*-q4Kp?k`_AvBdf6jr#TNl0f%);{1J(F)v zQxJ|xNVk&4qrOU?hQap@3(NVz184>%Y8s|UK?KdhZU&yJLA_NNlFjgM=metYpy%;7 zw%z-5C6?`b^~tZ+73BURGW!Mx(7`NE*b|1kruJG9#1NO@o!rRw2crnZ;C`h0>R){e zIeQ*df)h|%)8hA6t5p6X{jW;VI7upPx_yMX%*!>-K=P84%p4A&$_M`Myv?Gs*?zWs zSwMIj*zAiV8?DcQuU+1c3^a26@XB_Q$@z7w9Y#H&88$^o)s~V#iTda-&64${Ri7h( z43)jPt{ipysfNU^spq_Y5^@#GxY@n?YRzK$I-|JJnn}$!rst#kxrJH4 zu*lghc&))VygtIWN0;s>Gf)7hXm{w63R}OMJxqMjs5JuNVV$Z|b=9~B+fj}SV99Y- z)J1FQtIv`dM}GXPnlKWZ|Dtl;H;v8mv2vL3#N?#hPr}}_W z|6w|P(5xf7bk+N1$N~a_4-__maIkgarQBleJLuyL$`?W~EQDkc6-o0(0DwCH}ok70#RSduvVOzbEMA*q}@}Mtmo|JbIzpndCY#pCW zbsz%%TQ#jlKoRed3Hypy6#b#noO3nwtF$7B0CTHs-ca$y05LzbCC#KM{fkH-=lV%&K5x7Uu2zi&AlW7^cB4W=6*FpWWDp6^G)IW!TejjzR zQS50jAWM>C)UFyGB>#z*BqO+mi0E4z)wU4dDBx9SXM- zC#^bX>_bVnE~0O4jFcoF)X^(saXkwnfWKJ`MHyk8$+wS&Le7Rx2sDWvZ^C#azOx>o zhrr4fD&m7jvlx#E`yYL!t`Ak+=OoLVvcW9{4^3=iXGv69iwxBgJb&+pa=7pI#4vPX zW5F|^9a)R`@Ej|ySALaCO_%;3H*B24NDlA~ykCD3g?6sGX@9tg^t;);0{wmi&tGwx zElIiQE4c_a=+BRWu((LU|C)T`aQ!UST0dAN+Dz;Zbvpm^=5$krh?~k&PuPJcZ{>ru zf1veYTJC-T4@SOlRfH*8Tx)#F$Bbn5`t;#jE2`{egSSZ2GZ3-F%GN)DBQt;-VxeA2 zDX0XqH>7Dh)d^)IO3$`J<)c@OOVaYtrl>4*0jFN@aaR<3`;KjeiC$QY zY+l0(k<*3Fd^8XD)#eJUH1=K=swE^eSB?7!S0kpJc&(#R%~hP~!hG~BW67L!uEE)n zNO<)^;ucgOnQ!#B17Xr%YVr9uTblp*H4O>UgcXrR+0dVh?yXaJWlZFMKKpi`eRJ>m zU7;kaVkIT9?;4B!N$I_4#jfQI$ZQS|UM<$}W@zSou0W-aEYuKXvM7w~;DqAgkK@B7 zm}<09%^tMKvLaS{8)nocRbicfd}%9ZnNu?%O?hnAXsB^b3|WaNMf^6m)SHvSUCF}N z$FzXbhhX*PQmc0H+`LMFIB1E}b^s+lXPk67llIF|jk(0yP{HkGn#=`Jq?QbnQt7sr|!Fy#mZ9Tw{VI+X*T0<^sN-d_B)8w6kln%%vbz@sS)ngL>QKkiw@s z`9<%QrbKXA==g{=h;-#92oJZ20%B=LzK;0d-mYRFO9O0^@T-_&MCvw}nJ7g_G zb)b}au05~ET-`3_Q11Vn;UfB+1-A70WuJx7uKK@4QKkmaEeihnt*E>pRHeNI!pO{~ ze1g!mzK*CK`m~wtPeCR&vxJ%ayKZZw_O%8S4^^V|Ytryw%w2NM7jps;u1wFHnQ;k# z+d%=q(Bz>O8BR`>tf`7^@&l;&R2odL3>B-dXmrl{KU&HM6951U}zf&7&lYIBQxCb}YhpB_5am^1hoNx?#vO zZTSp?(I7`%ZeE!i*WG`ZQGtfb1nVyuhR60V0CO716$cQFpPio9Z?E&C*29O+ku>(4 zJ&ND@9fkq8qt|1L|5tbK8P??Xw2RY2k=`-1M2d7okkEUiNK-&U2a(=FSE_W7K%^>F zI?@HCi1ZSQfFK>IL6A=9f^g#dx_|H9`~T&9_+RH-*U87^d7imvX02H>nfqC*r)nuf zY2g}*LUwsm>-I~hs0z?{iIEC6zQC~{Ynpkj_>+}Wc)f3)LJFgOUIg^#krn0?W-2Hf zFmZ8Ck7F+ncUpC5dKnj_qby$zFrpabj{sM(a88e%#d-je`|DRw@_7^d;lS<UZtTLt&4uQI;WOUe(!n}6krIfn~4E7JWZ5XDj^z9NVJmMK&2eEazTYEEZQ9$ zYArbq32{H(sB6~?Iq-xVf4#X$>?H64mk-KDV7&lr@)?f)7&1PchkeNujQ+O2h$n?; z5gXlG=A=zg4$F^3ZKqsi2%_I???s|gFY`25vw;kBh%2pq~(rzcd5`u#h?Vsjp z@^NoT#3Jp4N|9NDYRb={)Uo4mkoo&>lR?w{eU?MOQ&;RTIGT)p5&^s!=tI9$2;RZ0 zF*mgj#A@NEv@MqYd~1{!vJIf&x<;eoTGyuz1T=VElFR$V3nmRrZ_sl#hl| zYv1WRNc9qy@A|bziSIsnGe@?%$|v){QqWbP;#Ot5-tz*AYEDEKP{E0f2Cg!NoULD3 z9idu-SgO#seT{ge7Qhbt#$at1DGQD~TGkuE}JOwr_ zHsVbLL{o1EeYE`WOJol|+rDRDU+{$~t?|=fB~FtJxH<+ zJ(MTWv719ybVwTQsL7M0@f`X z6lArFNyuD*rcJ%azI!q%yenH~{9~}&FU^RALe?lKg!pr_lY8+y){==LfXzTWBBI&! zp`fC|8~LLwkDeIQ+uKFDNy5p0EhE6tyJ!^ntP%;GGsEX%tE{(syr|=naMF@W2 z_+sf%bXVE;sxpY_J*`9o7++MNK-vbX_>2koiK-r)K$feXTw6$=Q0XX3di;30WWDqx z7cJ3>Zn#fHr?&?yrKYKgw_?n-49YKr(P64Cb7FoqAB{K8nmNAjJ8pQVAT0*ORw|W@ z+RyEZm$=ZVrSF|2mHnZU89W9L5saqP7Kd-R6TPS~8z`Gzu!?!~EV4=}PT!ys>ExbO z4OTT;^?E~W-@IDGcl!2Az_EfL1xb_Fuoa4dAi_--<#f#>yK#oNk#D2@OVE#fY=?A~ zs{~)kJk#QH0vUfFtN%kM^mV&73W?hq_v*4SPeUT-|tl* zSH*BTdKcGehC;M&xnwb zg`DNH$`n@%l+H4$iNxRAuDD|vu&wta)s3&C6B3*mN3b>=m9OiUG>Ym*hE-53CB}-w z174AD8vR>-8(3JdYg?B6G)N>9Zlb0wt946;=TIU(*|uL+$}nvB%V! z@MG8k_680((VZC=;Ljr>yVII0Ku6mM;R!wWQCT9T-}p#(4^nvX!PmSoqzS-jdFg(j zcx#hWL=Je77NhU%J{9`_NfaTo2Pg8zc2z$0QW!~Lo{(4j0iH^3qo{v;9O^{) zg8`a|nTv@VD}|AI=Tl8yjCE)amllh*&c`0c;3#+x66nBd;>F z^<&+OIDB1BGUSHjgRKixXh3LPI;Zmafbv__n|ry|(5I4{DC*+2+9YIR+KpSSheS@k ztuP2c{CvxF>DOmXjp1t>+y*QeOfCxtncgLeS9!r(mVMtab% z>v1aAfkLa|Q8YcfT?XHw)=K8m7(xmbt2I>IRCQ@zV^a@<2b(7UsZRxmFwYazu4`y> ze{I0ex1|v>7jylFw;myk{hgl^x(@IApzn?5fk&>lABre_Wlth`hR#lmQ~AcWiZ2S+ zTfuV+cHR6&?S&Q@(;F)bM6>gircC9rtY?@c_w~XgG1?wi<@AkTfk9LslRyQI%#;-C(mC6|^5fime_J;=7dB&Z&+rC!EbR-Iski zF1Z1^6T?_8sNl)Ts&nFh zU6Y$U$b5J$zAcGSRydY8M6TbVg=_5oD8B()RgS?eG#yUA-g~=%WwIEqh&Tu2D+mG< z>$uV6UIABsyTW3kGw`q~F7KL%UHHb`9)XcJSWI>v9IRh)Z764}>2e zoNl@Q{OujXfPn7e^H%Q#>9QVb5nA{aYmR)_ZF>WfBd%{}L#_4s)mZ z*Oep944L6Ap@@;3|IusRluY~IXpE;`BqiP(JvbM$kEi4!N`^yht}>lE)KTZYf7eNV z3o8OcBqbM2RB`aOJ49c|ZYal7P?7|4{SOg#W>^sm3VrJ%x#DukOu7SWFV8(I33)iz z7ylum{I`gk8)uiyb-htE_!>=_qQS%UOSjOM(*F?QjTPaK{|_6CG+{N%4E~3R+TSAn zWlP9!%?|heA!72kh^XewdF-j#_YZhkjsz~xb*&TE$I=p4^fp@4c;2m%=}jp)3WiT^ zR&__cP7L0KPe%DuEJPFh=)4qONiNBqjB4p(zAH0F+CAJY*WOK;t2ok`gSDKh&=gI+ zAF;?=8hPA7SSyXU;I@pW=!o_u!EC5*Js549_RtEkPXpp6+>D)pfYK#X=0Kz%Y1%D8 z>mGdj(c1lW<(Q6CG96AHgTX00Cq1&J8=TRr)8QKPL}Hx6A59+{NG>fFPl~DVbv&S< zgqKf7cij&$cGI4}EqSWP<@VnXP+UU^M=Q#NH-Lk=Pgt7>^yG*s9ls_gGMWvC+7Mkk ztI%0yT-Nes-|K>m38@-`q+lLD_^-`%TC%}5(upg5e{d*flQr&JbW6tXiDT?=oDLcj z^L3jd!&^4s8PsV9_g#aGL-D{2zb1Q-i??Q!jeWTdGbXp91pLFYBjmBeTqNxln&6AN z3)5Fj;h8LSA;P7n9cuyqwDAlHGO|49QK2Xup*bEGhC8u$0tR`SWox=(WSM@l=lwAy z6xg&4Xk;>wKZ3>L{mPB4VR^>Ici&95mR*kUQD$l6hg4-ji?~&UDPR8JEPsVZ$fQS= z`W;$K`Cy=A@*|O@`oyp~LsfAC8J#Zt^M01PPi5S_5+T~H@Fc#Z&ql(ehAR`l{O zy*KaOTnuFr>Y2+kqg>s(H9kHSM%Q`-vM$GCB)*XsSvPs7WKFXM`yc%coCf>V(JJtK zICUM&0BW1Jh)7&F8|Lzkn4Ny$qZjhlp0t-mQ_8o~GB8iOit~D+d!IhnvhzP`Nn@pM zHY<>LtMYPA9wg#1)>sSZP#;``SdBV(&*0HMK@t}5<|fyi?4a5L&6t=n;&*f&DtV)j? z-V(4j3=^(TM&FlWyQB$8yxUs&wj4Iz1BL^C`5t|xak=Mr6r#z0 z+)eZAGE8Y9*KTjKz*WR+ zhStg4pbHGxA?CBYt%V$X)=O;7y2hmH*Fg-Wb-kG|MiJ^D?E&k;z=kH)abs$&5m`af z7(gXI9ef|bG}cHD=JV(LCg>vHvW!X+;Fsqq=&R=4@9t7U&?+P2puw?3vSh*7uw^OK z5Du*K&n!}GrzZ7(c|@fMVdz@ZCp~YBG+2a2RT5Z!+9O5-6A{#VX8cj^LrAA}fp@R8 z=g>HB-O*zWUdp-)Ynz4iBxKsXKlS^`pbM;Y@XWK*@Rt9?it6MnDsU2t^Wt`FixOm# zN__wk^6=SXsXZ=&jx;cQ@amWi)c|fwP(}dXy5Pk3+xD-$e=^&wdYuIJ#lv|Px1~8z z#f4q$%3>Cs(8Wv2F^Zp@3gfP0MrLK&Gmx(nm4Os`Gt^Wap9PZ4^U_RFWeOA03y4eJ z6IKYrcocJPQX;N^MzU#zgYrEBTeHfthybpG$reXzVS=pv&+k7kPhOy4P_-N?< z8DE#gACAT+pi)P9utNkIEi_7n@JlwM6R-&5@)4#y)Bq@g`~7|zchIzjv{JS&B;h^z z=>6A>I0`^spns@9goW}n>CekkK6Ze&DJ}RN4jlaz}ypxu8)0vMr&C1%r zyzO3U1slzMp|AAz?m`X@wGMg!x#U=Lxzlf1fcN@pf~%0%cob6boEAGCg_b)z=lciK zJ>R|xl0bYQ$<{IgK!1l!#n=OphHX+xH?7Q+sZa7wg}>J@pa*)5EoC_p95vx01MscT zO6sJAiaz~mPWbb*5_@_`Iv7fJ8(TwbP>%|bFYblBPP9o=&hrBoLwk3VJbW^$=Xg)8 zq`SJl40jV1u(`4P@MjEXdVBb_UieiZ z**is{h{@VGyh2@z)cf~)YsEQ*n=^GVn=7GVg#{&(a#{I#H~^B9n@{G7UNK8Dsy1go z9`ayeW-|f(Sra-upcD!HKb{RDx16K`39hq&=TF~9m25lQp;RKnbkFiWbeUR z4^NfltNoVm}P`a2-(BdV%< zn=$0tm(iSm$-?gKCTues;>XXgj0Jp`=FJyKjs8QAGBU})hp81oba?yUlJH`)xTsV^DW&UXZ4H*!muKw3L^EyN|DE7_w#o$`q#T%odFpnL zo{q}=RWkf{lD=Ol`Ts!n{eQzV|C^;toEu3KU6ty$H8c`<>h$yM2IfpG5jRrO;tB=m zh|~G3z_R|y*D!_#=5#!7`qQJ?w5WzrD{)QOJDg@Da`H_(efqV4+2f_j`2yWwy`)to z>~2rT>_V^>BJZ@M-&KUURd)ro5yQYxZNJ+7Eq>DNk)`j(v82cUo6U<6i_bS-`05xO zij2ToSpL?xICH@sS9JA8MEwAij8Kh$bdrQDfFiUmwCoFWz#8>4S&F-@FFdX=JUpjN z94}9^_Xey#L!TKcQQX)pd)G=TLy(<1==E8IlmdmB^2Fuo*;9(7YK?tJ4KE=yms&z; za-MQt>gQpP5ZbDWR&OD)O&cHG(fiZ7@Z_$uk|k`_Kw8-m{qPNM$z=DG-atvOWx&x{ zM=jwkmWh>oZuK=*LP6CumGQ-EcPfSJ-?C4J@kgztj38vrf{DMR@*l*7u=m9aJat4| zf?=POP>L_J1<@<-8}V*;?>AcZ@QbAeVd> zdI)}!j@<%vt;G%dFYXp2ZwCOyQCiQ4^Pd(*(-r<#YICm_xNs<nkpWz=%|%iya;nI(q%MXxnU9DyVgB)z=o} zqPWMiqZCr^f_fqn*(Fl($mbjWj#`OB(|@FPiEV*)LNDKe~(f;&ippDP4KO%{RPS|$fkdX1W>XdBU(%;N;4lAS_=a#hiV zw|P>6!|cW6Z^G_Q6dt$9&VP^2Jk`lPU?0O%bzP-=rjI@V1ZPR*Qw$ok< zAfs!Nm^3y65C)bo3vk0x`{RZx!*)$L9_WoswHr#%w9iy4#8ef1?T2j$e!~Js;t7uDw0}@nppu^r&$@?`pqd3!^mqhfcrRM@ zo#P;vYO3V{dQAzGIj4)JR^}V`REq1sDIgaD&}os^6UyvZX-B^e_Hv%UQtDBWtqJV=Z*wcz?)vl_S}{@vv00Y*=EX z$oD~fFq{_-Bg#Gz(LrzIEERj!lpr#rTNqU#{$!Uk~q=7)@&zjS}TOkmF2 z9~9@`w|Zn?KXZ4xHn!!)SST?hGP-!6?tWe~!RxPn1oZ1&N}zX|4yN~S&zRaL!_!WL zrXJ4t0OB~;PAe1Q3ue-;9JWetJPM)~l??xs3J0?KL(j4IqYpmHUTzw;XbY`H-4 zt`dhfsjDLAsMWxk)*}vj?;Akg3`{|79aDR=6e~GK{(`S#%Q3p39 zc1}KPbH$;SJG52^AP}-It(AgjyuIODPo~zClC@Q`>HR;PX$j6*nImbD(skc$Jy-LS zBu1x|TLOtoj2g)~;Hg65x4F7#o_HzU!?CUjL3tC0exV>6AXRa#oWuB4TzlPO%Xw>6 zluo`jZp~NqV@B7KfeJF$w#goof<3jo40ul2O%0K0KPwHDZX!@WkKelRy8Xm;)KPm% zUKeenc~eC$9~I;F!>*%yibBR4HF!S+RIEBGF`o`vmj+@ZO9}2_do?Q(%uuDzAL0?? zY^QiGgUpc#GHRR*-p!_X{uXPz6!mtE)`x_Cm4TbE((5`Y<$67v&20oDNjPL`Cs8aM89ON^pUg59Wbe;x2 z)|GpJ{@%R9SnJr62aZjD>ZqNY?lCIpsPR$Mx>qkQCi9dX>ZzvnQhO5b8jdE%V@nJn zbDpax5Mb{7`N16+cdM<|&k*fGeVz&8Sq4(K41K}P6n~_Gh_y1Zuhd5$o6(&eD5yE( znAhLYM5}0o7zyy^aKmexfK_cl_u(zMsM1Y7Yn5yZX8sxHRk9;3N)Gr3`MsRf5E%J( zk)tXVQoaftRetv#qOMgilINjz^4KkhAPDQRaXN_I=oi1LYtC(qe$IM@L;-3&`f@T8 z6pV!Ri@#oArpNv!uZb=Sy+_e#EgIvL&uOq{OOu<9n{AS@XXQUJfStx4b1 zU`&}SZwEq9L0cRHFQQw1vO!!| zXlp12Gi{nJW>Uc#WXY{B`ALO+VCn|^vptw_CNVDaS+ByL#j}ho+f$qfDON(zJyM(c z5@9O?{%nb)k7)lT#d87VxD5v5cCm+~T`!V;(Hs>TY#qw~y<;hfoKD)ZD5}i>{XD4l zAq#TqTA50Sy5-K$-LC1@RzV&X&os}$>sMRz}8m5eNQ+cdD~8^Sdk6>SX8^yZ6Yo#X2& z1cxQ7UilV^gnn-5Xg@9X(t@?=75*-66+s-4zgZA+y55EJ7dcbUJ4&86^7qux39-QY zYPRWTl6c40fxkqS@5?#k?|YDp?hZn>eRTm{2)W`tFP2!?4hOBH8m_`f zdT;}HBMH~URLpHIgkwG0Ib5AUOg~kTtriqnpz*->ZY!y6$$r1NSwCp?)Y&z;Jt|ls}HS?bdiy*37jVLEs@u z`YrX^hq>5{0>V_sQ94x$_;f!03Q#W?Pj&RB=7@9u`f*5f00Um$BclodHP+;XXj6x8 z^CWXw|9zd{42J$f@zJHtym}0@5Vc4)AQMjQues?^#dh>vN0Y7Y$za2Fab|uK!1=q_Y(tQl`Gx)JD^tDWPf5X{6}md zfb!X4F&8E5{aV}!z;LHDQYbpJ2XrNfJk0fiqF3lODgPsgac4@do2f;xY}lBKtB(_dDlHR3U#Vo6N@NoKzY6{#44DMKF2bjwxGne6Cl|SlC~ByoX+J^(oOHHH`#h z{YEq71^q`~2|lm0@DjL1?GF%Z4fqR)ZNiBrBH|V(O@4LoH1}RG43o=ca$?`M@%TOn zL-91={6B!$cG7@k>MBCPy=i-Y>AC$S=u1ta3YvB33?us&CfQkG2*ayr)2o-li(w0w z4*g-N&NKPH@@zO`P}1oKYtR3N2ad(lbjfVd`ZwUJzk&OT{1XoviwA!GPvCLCf$v%V z6HotdJQ%xw0B2;y0$=F(C!SZo@vM#f6Sx!>c)%Bd;&q8n_!SGPQEyaRv9{aCX3|lH zd!1Ka@ADtK-e4nhsnDNfsiz62K3^4Bw`u;S@AXb0%Im^`M9n~CyohNutT6u*-cu<= zj8qUlt0TiXc2wZju8jomn8m^KWhez(-I;rDWT&47Ph3~of`L(jJymS$oi5P(OSHAudSkB%=^IzR3~nZ zjtk@dqwtK|Y#Y-m3CcPlB_Bg?;(LzVX;S#y$xg{dkOIp2^2{5%L)n$ZY#V(Ok-nTg z+w^@p~dHiK~R!I*Ds39i4j#dS&nr0Hws15$KX^Vg3<;my84@~DNuVhobd8&<@ z7P4QV@x;l3Ws<2ZgdAzrMn+k3G1hLO|R`{|rKL@V}D z1}hOo{bf0;5qj3^L;0r~hDDB>BtNS?$Jbhc`h3LuMk>4GEQ4zIyZj~Cc)3lMPKroh zy5L;LH)!QPT~3Iic_FKcU@PQ#yeYAY?~F*RIlopo_B^PuA0HO69XmggMC~>i^!E;k zhUFX8|FB}hcBr~ORH3?F4fW#xs7Tmzr&X)|p5e_BRPrLlkKle~0oezRJ}`txe<)8;EM* z6x+A<@u%jjv`=b0>`sk12T7Lh#!^GI)Cd!-Picx*Gdc%mLUA53tSVj~6b>QK8oLc%9irL#^MDG9F50 zhHN=SDgDbm1!C*llXGHyc75fe3h|T;pzB}WwyjPQHx`d6IJ;eVJBlhiJDN81UiXD1 zAU2eOb_oKfW6UZO``bS`0AhXFP9yP=`B&O+VENlb@oer6g%ss_BRRL=e>|9dicFY! zwy$DQW{_cQKGI&VFv5*!&nHu3Hv;W-kcWw~W;(HlbEf4}Pkn5rNr>D7OwRc&8=~Lu>niOhNK4n+A;i=; zwp9A=cOEdk^aRssQbqJ`p2L__+KsV&tv zU-;1Z7ul|+IiE8z5NgY4p_!OsO@V=b`KsOYBpTR=?e3U&5XZ>)!}d8?%Yw2sLG1Z+ zbuRiFYN0(Bh>+n94Aag+J^Tv$bYE8m!}ksuz&r728t4d6RjT1#0ZH#L;n&!*>a!xK zuRR$0*W8fd$N0kp{fn$yU5iyPL_elq@Q6vXXgOH@Xqi|dFhhJu*{(WOSctt0Vbw$1 z`w$~-NMZ1su=i%`FE~KK`e+KZ_J+pR@V3+tR#R{l9=5rY)~-zQySbwt{8jAU>An>@ z64gPe{KV(({?gOb+vZS{G-~pNp#QtI4 zWDoX5^<$R@>)Ap*h!q=3_*3i({uBCnql+X3t`lX7oMSa2^#WIs8UgP$3z&XU}akGt>o=1+W*-vdWc@1NcM z(7VlxVc?l6-U`ytx_X1M3-IK}G;F7X?o~6%c&SBbAiMHHBV^;`nZ-bTLJbpN6qX75 zBnXlBnwx8D%XYWtDd)p4hP9ip^XkkPnc;m@?eWBiu4w_~D~rhQ9q$;T3!`&F??`?) zWjSY`R#e4qB;@?)mN1%aT9h$lc8wa_(J8P7GQ_sf4I^Vc9^dy#ihLd#hiS$(vkI26 z@h=)Ad|Gl;MGvWTT?I-8zTpm|g^IhLeS+6v8#bDsvmYuvjk+T3mGm554MED!8N z9-XY@0r&~=0Z}WO4cZjL=XV*IKb7M9GM&BTVEm(%krVnb{T|2gFEZ6i?sNj-ck@s= zoge6-#RP|!dnT8aDH|cW_E=9?m>TU5vF8V%b6aplZ)i9$h2p2;)7*?{R&CpWKLFT! z4tCnPq&mB-#I*Wl+CiAp9$ap<54>F;D&+g4*MS-TjM~x<7%~jDczvRL)UXz&s2HrZE z;WdtntV&`~)a3V06Em|LVE#~BSnMDXb#yfYz;p5*{uA)eRE3K6Tv#60JMC=o5Yz{r z*wh*FPdFFX?YF)!veFx*oL+`mT0JPDC$C!in%n&?HNNgAROYFQxp&i4n!&O^?Q_<% zmOqKXev@e{%fk71Ex5i!eGrnyDjampsEo$LvTX>)u+b1p{o((9eE{Ql{?S4-vEewQ){mXTDMeqHQ@d$gnVGkN%SsfrMmPz2>5$4>ENc$4)NdsPIRl6|qFH>e7E1bn5c@JnC9(R~)Xi krF{?Y;4k=gW^vcJJx7aub-H(_aIhbBRc)0@CCeB83qAjf)&Kwi literal 0 HcmV?d00001 From f0bd2d05f5f7832df4879822afb99d2096c00d48 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 15:24:54 -0700 Subject: [PATCH 0153/1866] Link to global-power-plants demo instead of sf-trees --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 2f0a7962..52434fdc 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -6,8 +6,8 @@ Play with a live demo The best way to experience Datasette for the first time is with a demo: +* `global-power-plants.datasettes.com `__ provides a searchable database of power plants around the world, using data from the `World Resources Institude `__ rendered using the `datasette-cluster-map `__ plugin. * `fivethirtyeight.datasettes.com `__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository `__. -* `sf-trees.datasettes.com `__ demonstrates the `datasette-cluster-map `__ plugin running against 190,000 trees imported from `data.sfgov.org `__. .. _getting_started_glitch: From 7788d62fa679fa87d3f34a3466295b0ae06598dd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 20:28:16 -0700 Subject: [PATCH 0154/1866] Expanded the Binary plugins section --- docs/binary_data.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/binary_data.rst b/docs/binary_data.rst index 593cf78d..6902af00 100644 --- a/docs/binary_data.rst +++ b/docs/binary_data.rst @@ -63,6 +63,6 @@ Binary plugins Several Datasette plugins are available that change the way Datasette treats binary data. -- `datasette-render-binary `__ modifies -- https://github.com/simonw/datasette-render-images -- https://github.com/simonw/datasette-media \ No newline at end of file +- `datasette-render-binary `__ modifies Datasette's default interface to show an automatic guess at what type of binary data is being stored, along with a visual representation of the binary value that displays ASCII strings directly in the interface. +- `datasette-render-images `__ detects common image formats and renders them as images directly in the Datasette interface. +- `datasette-media `__ allows Datasette interfaces to be configured to serve binary files from configured SQL queries, and includes the ability to resize images directly before serving them. From 4785172bbcb9edd22b6955b415cd18cd4d83f0aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 20:33:47 -0700 Subject: [PATCH 0155/1866] Release 0.51.1 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index f6e9ce97..2d949370 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.51" +__version__ = "0.51.1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index b9120c52..97d5d251 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_51_1: + +0.51.1 (2020-10-31) +------------------- + +- Improvements to the new :ref:`binary` documentation page. + .. _v0_51: 0.51 (2020-10-31) From 59b252a0c020d687259ab85e06f0636feefa0dd0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Oct 2020 21:45:42 -0700 Subject: [PATCH 0156/1866] Link to annotated release notes for 0.51 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c101a4ed..a10ccfd3 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly new ## News - * 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. + * 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. [Annotated release notes](https://simonwillison.net/2020/Nov/1/datasette-0-51/). * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. From b61f6cceb5682f9154ba72259c0c9c7503a605bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Nov 2020 09:22:13 -0800 Subject: [PATCH 0157/1866] Add nav menu to pattern portfolio --- datasette/templates/patterns.html | 38 +++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 62ef1322..25fb6008 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -9,19 +9,33 @@ +
    -
    -
    - -
    - - -
    - -
    - - -

    Pattern Portfolio

    From 7b194920702358b65739a6e8bd3adb765ffa346a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Nov 2020 10:27:25 -0800 Subject: [PATCH 0158/1866] database_actions() plugin hook, closes #1077 --- datasette/hookspecs.py | 5 +++++ datasette/static/app.css | 6 +++--- datasette/templates/database.html | 25 ++++++++++++++++++++++++- datasette/templates/table.html | 6 +++--- datasette/views/database.py | 15 +++++++++++++++ docs/plugin_hooks.rst | 16 ++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 11 +++++++++++ tests/test_api.py | 1 + tests/test_plugins.py | 19 ++++++++++++++++++- 10 files changed, 97 insertions(+), 8 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 78070e67..a305ca6a 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -107,3 +107,8 @@ def menu_links(datasette, actor): @hookspec def table_actions(datasette, actor, database, table): "Links for the table actions menu" + + +@hookspec +def database_actions(datasette, actor, database): + "Links for the database actions menu" diff --git a/datasette/static/app.css b/datasette/static/app.css index a1eb2099..675285c1 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -360,11 +360,11 @@ details .nav-menu-inner { display: block; } -/* Table actions menu */ -.table-menu-links { +/* Table/database actions menu */ +.actions-menu-links { position: relative; } -.table-menu-links .dropdown-menu { +.actions-menu-links .dropdown-menu { position: absolute; top: 2rem; right: 0; diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 3b89d68b..7065f2c2 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -18,7 +18,30 @@ {% block content %} -

    {{ metadata.title or database }}{% if private %} 🔒{% endif %}

    + +
    +

    {{ metadata.title or database }}{% if private %} 🔒{% endif %}

    + {% set links = database_actions() %}{% if links %} +
    + + Table actions + + + +
    + {% if links %} + + {% endif %} +
    +
    {% endif %} +
    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 13f6a832..5034b62e 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -28,12 +28,12 @@

    {{ metadata.title or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

    {% set links = table_actions() %}{% if links %} -
    - + - Table actions + Table actions diff --git a/datasette/views/database.py b/datasette/views/database.py index 3ed60f4e..9a7b96fd 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -6,6 +6,7 @@ import json from urllib.parse import parse_qsl, urlencode from datasette.utils import ( + await_me_maybe, check_visibility, to_css_class, validate_sql_select, @@ -101,6 +102,19 @@ class DatabaseView(DataView): ) if visible: canned_queries.append(dict(query, private=private)) + + async def database_actions(): + links = [] + for hook in pm.hook.database_actions( + datasette=self.ds, + database=database, + actor=request.actor, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + return ( { "database": database, @@ -118,6 +132,7 @@ class DatabaseView(DataView): ), }, { + "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, "metadata": metadata, diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1c28c72e..6f8d269d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1057,3 +1057,19 @@ This example adds a new table action if the signed in user is ``"root"``: "href": datasette.urls.path("/-/edit-schema/{}/{}".format(database, table)), "label": "Edit schema for this table", }] + +.. _plugin_hook_database_actions: + +database_actions(datasette, actor, database) +-------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. diff --git a/tests/fixtures.py b/tests/fixtures.py index d2ac661d..a48cfb46 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -38,6 +38,7 @@ EXPECTED_PLUGINS = [ "actor_from_request", "asgi_wrapper", "canned_queries", + "database_actions", "extra_body_script", "extra_css_urls", "extra_js_urls", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index cd2c8e23..4ac3953b 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -333,3 +333,14 @@ def table_actions(datasette, database, table, actor): }, {"href": datasette.urls.instance(), "label": "Table: {}".format(table)}, ] + + +@hookimpl +def database_actions(datasette, database, actor): + if actor: + return [ + { + "href": datasette.urls.instance(), + "label": "Database: {}".format(database), + } + ] diff --git a/tests/test_api.py b/tests/test_api.py index 3365bf57..1a43e7f4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -60,6 +60,7 @@ def test_homepage_sort_by_relationships(app_client): def test_database_page(app_client): response = app_client.get("/fixtures.json") + assert response.status == 200 data = response.json assert "fixtures" == data["database"] assert [ diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5e3d6dc3..3819c872 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -788,7 +788,7 @@ def test_hook_menu_links(app_client): def test_hook_table_actions(app_client, table_or_view): def get_table_actions_links(html): soup = Soup(html, "html.parser") - details = soup.find("details", {"class": "table-menu-links"}) + details = soup.find("details", {"class": "actions-menu-links"}) if details is None: return [] return [{"label": a.text, "href": a["href"]} for a in details.select("a")] @@ -802,3 +802,20 @@ def test_hook_table_actions(app_client, table_or_view): {"label": "Database: fixtures", "href": "/"}, {"label": "Table: {}".format(table_or_view), "href": "/"}, ] + + +def test_hook_database_actions(app_client): + def get_table_actions_links(html): + soup = Soup(html, "html.parser") + details = soup.find("details", {"class": "actions-menu-links"}) + if details is None: + return [] + return [{"label": a.text, "href": a["href"]} for a in details.select("a")] + + response = app_client.get("/fixtures") + assert get_table_actions_links(response.text) == [] + + response_2 = app_client.get("/fixtures?_bot=1") + assert get_table_actions_links(response_2.text) == [ + {"label": "Database: fixtures", "href": "/"}, + ] From d6257e3a7b9b1652386451756faa19d03a22cb69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Nov 2020 10:53:52 -0800 Subject: [PATCH 0159/1866] Add database/table actions to pattern portfolio Refs #1066, #1077 --- datasette/templates/_close_open_menus.html | 16 ++++ datasette/templates/base.html | 18 +---- datasette/templates/patterns.html | 91 ++++++++++------------ 3 files changed, 60 insertions(+), 65 deletions(-) create mode 100644 datasette/templates/_close_open_menus.html diff --git a/datasette/templates/_close_open_menus.html b/datasette/templates/_close_open_menus.html new file mode 100644 index 00000000..65eebddf --- /dev/null +++ b/datasette/templates/_close_open_menus.html @@ -0,0 +1,16 @@ + diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 611ba9f6..3ed67164 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -59,22 +59,8 @@
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    - +{% include "_close_open_menus.html" %} + {% for body_script in body_scripts %} {% endfor %} diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 25fb6008..4ef2c29f 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -62,15 +62,6 @@

    Example message

    Example message

    - - - - - - - - -

    .bd for /

    Datasette Fixtures

    @@ -101,15 +92,27 @@

    names, foo

    - - - - - -

    .bd for /database

    -

    fixtures

    +
    +

    fixtures

    +
    + + Table actions + + + +
    + +
    +
    +
    +
    Test tables description
    @@ -150,14 +153,28 @@
    - - - - -

    .bd for /database/table

    +
    -

    roadside_attraction_characteristics

    +
    +

    roadside_attraction_characteristics

    +
    + + Table actions + + + +
    + +
    +
    +
    +

    Data license: Apache License 2.0 @@ -257,7 +274,6 @@

    -

    2 extra where clauses

      @@ -269,7 +285,6 @@
    -

    ✎ View and edit SQL

    This data as json, CSV (advanced)

    @@ -278,11 +293,6 @@ Suggested facets: tags, created (date), tags (array)

    - - - - -
    @@ -420,16 +430,6 @@ ); - - - - - - - - - -

    .bd for /database/table/row

    roadside_attractions: 2

    @@ -474,16 +474,6 @@
    - - - - - - - - - -

    .ft

    + +{% include "_close_open_menus.html" %} + From 13d1228d80c91d382a05b1a9549ed02c300ef851 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 Nov 2020 12:02:50 -0800 Subject: [PATCH 0160/1866] /dbname/tablename/-/modify-table-schema is OK after all Refs #1053, #296 --- docs/writing_plugins.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 29fcca13..dfcda8a9 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -227,10 +227,11 @@ If your plugin includes functionality that relates to a specific database you co - ``/dbname/-/upload-excel`` -Reserving routes under ``/dbname/tablename/-/...`` is not a good idea because a table could conceivably include a row with a primary key value of ``-``. Instead, you could use a pattern like this: +Or for a specific table like this: -- ``/dbname/-/upload-excel/tablename`` +- ``/dbname/tablename/-/modify-table-schema`` +Note that a row could have a primary key of ``-`` and this URL scheme will still work, because Datasette row pages do not ever have a trailing slash followed by additional path components. .. _writing_plugins_building_urls: From 2a981e2ac1d13125973904b777d00ea75e8df4e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Nov 2020 15:37:37 -0800 Subject: [PATCH 0161/1866] Blank foreign key labels now show as hyphens, closes #1086 --- datasette/templates/table.html | 4 +- datasette/views/table.py | 2 +- tests/fixtures.py | 6 ++- tests/test_api.py | 76 ++++++++++++++++++++++------------ tests/test_csv.py | 6 +-- tests/test_html.py | 30 +++++++++----- 6 files changed, 80 insertions(+), 44 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 5034b62e..077332dc 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -156,9 +156,9 @@
      {% for facet_value in facet_info.results %} {% if not facet_value.selected %} -
    • {{ (facet_value.label if facet_value.label is not none else "_") }} {{ "{:,}".format(facet_value.count) }}
      • +
    • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
      • {% else %} -
    • {{ facet_value.label }} · {{ "{:,}".format(facet_value.count) }} ✖
      • +
    • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
      • {% endif %} {% endfor %} {% if facet_info.truncated %} diff --git a/datasette/views/table.py b/datasette/views/table.py index 65fe7f8b..d29ef201 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -195,7 +195,7 @@ class RowTableShared(DataView): table=urllib.parse.quote_plus(other_table), link_id=urllib.parse.quote_plus(str(value)), id=str(jinja2.escape(value)), - label=str(jinja2.escape(label)), + label=str(jinja2.escape(label)) or "-", ) ) elif value in ("", None): diff --git a/tests/fixtures.py b/tests/fixtures.py index a48cfb46..bd530398 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -386,8 +386,10 @@ CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_k CREATE TABLE foreign_key_references ( pk varchar(30) primary key, foreign_key_with_label varchar(30), + foreign_key_with_blank_label varchar(30), foreign_key_with_no_label varchar(30), FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) ); @@ -622,8 +624,8 @@ INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); -INSERT INTO foreign_key_references VALUES (1, 1, 1); -INSERT INTO foreign_key_references VALUES (2, null, null); +INSERT INTO foreign_key_references VALUES (1, 1, 3, 1); +INSERT INTO foreign_key_references VALUES (2, null, null, null); INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); INSERT INTO custom_foreign_key_label VALUES (1, 1); diff --git a/tests/test_api.py b/tests/test_api.py index 1a43e7f4..d6d683b7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -49,21 +49,21 @@ def test_homepage_sort_by_relationships(app_client): tables = [ t["name"] for t in response.json["fixtures"]["tables_and_views_truncated"] ] - assert [ + assert tables == [ "simple_primary_key", + "foreign_key_references", "complex_foreign_keys", "roadside_attraction_characteristics", "searchable_tags", - "foreign_key_references", - ] == tables + ] def test_database_page(app_client): response = app_client.get("/fixtures.json") assert response.status == 200 data = response.json - assert "fixtures" == data["database"] - assert [ + assert data["database"] == "fixtures" + assert data["tables"] == [ { "name": "123_starts_with_digits", "columns": ["content"], @@ -232,7 +232,12 @@ def test_database_page(app_client): }, { "name": "foreign_key_references", - "columns": ["pk", "foreign_key_with_label", "foreign_key_with_no_label"], + "columns": [ + "pk", + "foreign_key_with_label", + "foreign_key_with_blank_label", + "foreign_key_with_no_label", + ], "primary_keys": ["pk"], "count": 2, "hidden": False, @@ -245,6 +250,11 @@ def test_database_page(app_client): "column": "foreign_key_with_no_label", "other_column": "id", }, + { + "other_table": "simple_primary_key", + "column": "foreign_key_with_blank_label", + "other_column": "id", + }, { "other_table": "simple_primary_key", "column": "foreign_key_with_label", @@ -403,6 +413,11 @@ def test_database_page(app_client): "fts_table": None, "foreign_keys": { "incoming": [ + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + }, { "other_table": "foreign_key_references", "column": "id", @@ -548,7 +563,7 @@ def test_database_page(app_client): "foreign_keys": {"incoming": [], "outgoing": []}, "private": False, }, - ] == data["tables"] + ] def test_no_files_uses_memory_database(app_client_no_files): @@ -1203,32 +1218,38 @@ def test_row_foreign_key_tables(app_client): "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" ) assert response.status == 200 - assert [ + assert response.json["foreign_key_tables"] == [ { - "column": "id", - "count": 1, - "other_column": "foreign_key_with_label", "other_table": "foreign_key_references", - }, - { - "column": "id", - "count": 1, - "other_column": "f3", - "other_table": "complex_foreign_keys", - }, - { "column": "id", + "other_column": "foreign_key_with_blank_label", "count": 0, - "other_column": "f2", - "other_table": "complex_foreign_keys", }, { + "other_table": "foreign_key_references", "column": "id", + "other_column": "foreign_key_with_label", "count": 1, - "other_column": "f1", - "other_table": "complex_foreign_keys", }, - ] == response.json["foreign_key_tables"] + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + }, + ] def test_unit_filters(app_client): @@ -1593,13 +1614,14 @@ def test_expand_label(app_client): "/fixtures/foreign_key_references.json?_shape=object" "&_label=foreign_key_with_label&_size=1" ) - assert { + assert response.json == { "1": { "pk": "1", "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_blank_label": "3", "foreign_key_with_no_label": "1", } - } == response.json + } @pytest.mark.parametrize( @@ -1790,11 +1812,13 @@ def test_null_foreign_keys_are_not_expanded(app_client): { "pk": "1", "foreign_key_with_label": {"value": "1", "label": "hello"}, + "foreign_key_with_blank_label": {"value": "3", "label": ""}, "foreign_key_with_no_label": {"value": "1", "label": "1"}, }, { "pk": "2", "foreign_key_with_label": None, + "foreign_key_with_blank_label": None, "foreign_key_with_no_label": None, }, ] == response.json diff --git a/tests/test_csv.py b/tests/test_csv.py index 3e91fb04..209bce2b 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -42,9 +42,9 @@ pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,com ) EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV = """ -pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label -1,1,hello,1,1 -2,,,, +pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_blank_label,foreign_key_with_blank_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label +1,1,hello,3,,1,1 +2,,,,,, """.lstrip().replace( "\n", "\r\n" ) diff --git a/tests/test_html.py b/tests/test_html.py index 006c223d..7fca8a68 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -464,7 +464,7 @@ def test_facet_display(app_client): ], } ) - assert [ + assert actual == [ { "name": "city_id", "items": [ @@ -520,7 +520,7 @@ def test_facet_display(app_client): }, ], }, - ] == actual + ] def test_facets_persist_through_filter_form(app_client): @@ -801,37 +801,47 @@ def test_table_html_foreign_key_links(app_client): response = app_client.get("/fixtures/foreign_key_references") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") - expected = [ + actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert actual == [ [ '
    ', '', + '', '', ], [ '', '', + '', '', ], ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] + + +def test_table_html_foreign_key_facets(app_client): + response = app_client.get( + "/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label" + ) + assert response.status == 200 + assert ( + '
  • ' + "- 1
    • " + ) in response.text def test_table_html_disable_foreign_key_links_with_labels(app_client): response = app_client.get("/fixtures/foreign_key_references?_labels=off&_size=1") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") - expected = [ + actual = [[str(td) for td in tr.select("td")] for tr in table.select("tbody tr")] + assert actual == [ [ '    ', '', + '', '', ] ] - assert expected == [ - [str(td) for td in tr.select("td")] for tr in table.select("tbody tr") - ] def test_table_html_foreign_key_custom_label_column(app_client): From e8e0a6f284ca953b2980186c4356594c07bd1929 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Nov 2020 16:02:58 -0800 Subject: [PATCH 0162/1866] Use FTS4 in fixtures Closes #1081 --- tests/fixtures.py | 2 +- tests/test_api.py | 28 ++++++++++++++++------------ tests/test_internals_database.py | 7 ++++--- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index bd530398..183b8ca4 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -488,7 +488,7 @@ INSERT INTO searchable_tags (searchable_id, tag) VALUES ; CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, [name with . and spaces], content="searchable"); + USING FTS4 (text1, text2, [name with . and spaces], content="searchable"); INSERT INTO "searchable_fts" (rowid, text1, text2, [name with . and spaces]) SELECT rowid, text1, text2, [name with . and spaces] FROM searchable; diff --git a/tests/test_api.py b/tests/test_api.py index d6d683b7..93097574 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -37,9 +37,9 @@ def test_homepage(app_client): assert len(d["tables_and_views_truncated"]) == 5 assert d["tables_and_views_more"] is True # 4 hidden FTS tables + no_primary_key (hidden in metadata) - assert d["hidden_tables_count"] == 5 - # 201 in no_primary_key, plus 5 in other hidden tables: - assert d["hidden_table_rows_sum"] == 206 + assert d["hidden_tables_count"] == 6 + # 201 in no_primary_key, plus 6 in other hidden tables: + assert d["hidden_table_rows_sum"] == 207 assert d["views_count"] == 4 @@ -512,7 +512,7 @@ def test_database_page(app_client): }, { "name": "searchable_fts", - "columns": ["text1", "text2", "name with . and spaces", "content"], + "columns": ["text1", "text2", "name with . and spaces"], "primary_keys": [], "count": 2, "hidden": True, @@ -521,14 +521,8 @@ def test_database_page(app_client): "private": False, }, { - "name": "searchable_fts_content", - "columns": [ - "docid", - "c0text1", - "c1text2", - "c2name with . and spaces", - "c3content", - ], + "name": "searchable_fts_docsize", + "columns": ["docid", "size"], "primary_keys": ["docid"], "count": 2, "hidden": True, @@ -563,6 +557,16 @@ def test_database_page(app_client): "foreign_keys": {"incoming": [], "outgoing": []}, "private": False, }, + { + "name": "searchable_fts_stat", + "columns": ["id", "value"], + "primary_keys": ["id"], + "count": 1, + "hidden": True, + "fts_table": None, + "foreign_keys": {"incoming": [], "outgoing": []}, + "private": False, + }, ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 8042cf53..e5938f3b 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -297,7 +297,7 @@ async def test_get_all_foreign_keys(db): @pytest.mark.asyncio async def test_table_names(db): table_names = await db.table_names() - assert [ + assert table_names == [ "simple_primary_key", "primary_key_multiple_columns", "primary_key_multiple_columns_explicit_label", @@ -316,9 +316,10 @@ async def test_table_names(db): "searchable", "searchable_tags", "searchable_fts", - "searchable_fts_content", "searchable_fts_segments", "searchable_fts_segdir", + "searchable_fts_docsize", + "searchable_fts_stat", "select", "infinity", "facet_cities", @@ -327,7 +328,7 @@ async def test_table_names(db): "roadside_attractions", "attraction_characteristic", "roadside_attraction_characteristics", - ] == table_names + ] @pytest.mark.asyncio From 253f2d9a3cc96edcb47b33c6971300d0ff15d4dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 11 Nov 2020 20:36:44 -0800 Subject: [PATCH 0163/1866] Use correct QueryInterrupted exception on row page, closes #1088 --- datasette/views/table.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d29ef201..9ed45df1 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -19,7 +19,6 @@ from datasette.utils import ( path_with_added_args, path_with_removed_args, path_with_replaced_args, - sqlite3, to_css_class, urlsafe_components, value_as_boolean, @@ -1040,7 +1039,7 @@ class RowView(RowTableShared): ) try: rows = list(await db.execute(sql, {"id": pk_values[0]})) - except sqlite3.OperationalError: + except QueryInterrupted: # Almost certainly hit the timeout return [] From 5eb8e9bf250b26e30b017d39a392c33973997656 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Nov 2020 12:07:19 -0800 Subject: [PATCH 0164/1866] Removed words that minimize involved difficulty, closes #1089 --- docs/changelog.rst | 8 ++++---- docs/contributing.rst | 8 ++++---- docs/deploying.rst | 4 ++-- docs/ecosystem.rst | 2 +- docs/internals.rst | 2 +- docs/metadata.rst | 2 +- docs/plugin_hooks.rst | 2 +- docs/publish.rst | 4 ++-- docs/sql_queries.rst | 2 +- docs/writing_plugins.rst | 4 ++-- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97d5d251..34bd95d4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -239,7 +239,7 @@ Better plugin documentation The plugin documentation has been re-arranged into four sections, including a brand new section on testing plugins. (`#687 `__) - :ref:`plugins` introduces Datasette's plugin system and describes how to install and configure plugins. -- :ref:`writing_plugins` describes how to author plugins, from simple one-off plugins to packaged plugins that can be published to PyPI. It also describes how to start a plugin using the new `datasette-plugin `__ cookiecutter template. +- :ref:`writing_plugins` describes how to author plugins, from one-off single file plugins to packaged plugins that can be published to PyPI. It also describes how to start a plugin using the new `datasette-plugin `__ cookiecutter template. - :ref:`plugin_hooks` is a full list of detailed documentation for every Datasette plugin hook. - :ref:`testing_plugins` describes how to write tests for Datasette plugins, using `pytest `__ and `HTTPX `__. @@ -277,7 +277,7 @@ Authentication Prior to this release the Datasette ecosystem has treated authentication as exclusively the realm of plugins, most notably through `datasette-auth-github `__. -0.44 introduces :ref:`authentication` as core Datasette concepts (`#699 `__). This makes it easier for different plugins can share responsibility for authenticating requests - you might have one plugin that handles user accounts and another one that allows automated access via API keys, for example. +0.44 introduces :ref:`authentication` as core Datasette concepts (`#699 `__). This enables different plugins to share responsibility for authenticating requests - you might have one plugin that handles user accounts and another one that allows automated access via API keys, for example. You'll need to install plugins if you want full user accounts, but default Datasette can now authenticate a single root user with the new ``--root`` command-line option, which outputs a one-time use URL to :ref:`authenticate as a root actor ` (`#784 `__):: @@ -572,7 +572,7 @@ Also in this release: 0.32 (2019-11-14) ----------------- -Datasette now renders templates using `Jinja async mode `__. This makes it easy for plugins to provide custom template functions that perform asynchronous actions, for example the new `datasette-template-sql `__ plugin which allows custom templates to directly execute SQL queries and render their results. (`#628 `__) +Datasette now renders templates using `Jinja async mode `__. This means plugins can provide custom template functions that perform asynchronous actions, for example the new `datasette-template-sql `__ plugin which allows custom templates to directly execute SQL queries and render their results. (`#628 `__) .. _v0_31_2: @@ -1881,7 +1881,7 @@ as a more powerful alternative to SQL views. This will write those values into the metadata.json that is packaged with the app. If you also pass ``--metadata=metadata.json`` that file will be updated with the extra values before being written into the Docker image. -- Added simple production-ready Dockerfile (`#94`_) [Andrew +- Added production-ready Dockerfile (`#94`_) [Andrew Cutler] - New ``?_sql_time_limit_ms=10`` argument to database and table page (`#95`_) - SQL syntax highlighting with Codemirror (`#89`_) [Tom Dyson] diff --git a/docs/contributing.rst b/docs/contributing.rst index 375f6b89..ca194001 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -19,7 +19,7 @@ General guidelines Setting up a development environment ------------------------------------ -If you have Python 3.6 or higher installed on your computer (on OS X the easiest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. +If you have Python 3.6 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. If you want to use GitHub to publish your changes, first `create a fork of datasette `__ under your own GitHub account. @@ -27,7 +27,7 @@ Now clone that repository somewhere on your computer:: git clone git@github.com:YOURNAME/datasette -If you just want to get started without creating your own fork, you can do this instead:: +If you want to get started without creating your own fork, you can do this instead:: git clone git@github.com:simonw/datasette @@ -47,9 +47,9 @@ Once you have done this, you can run the Datasette unit tests from inside your ` pytest -To run Datasette itself, just type ``datasette``. +To run Datasette itself, type ``datasette``. -You're going to need at least one SQLite database. An easy way to get started is to use the fixtures database that Datasette uses for its own tests. +You're going to need at least one SQLite database. A quick way to get started is to use the fixtures database that Datasette uses for its own tests. You can create a copy of that database by running this command:: diff --git a/docs/deploying.rst b/docs/deploying.rst index e777f296..3eeaaad8 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -4,7 +4,7 @@ Deploying Datasette ===================== -The easiest way to deploy a Datasette instance on the internet is to use the ``datasette publish`` command, described in :ref:`publishing`. This can be used to quickly deploy Datasette to a number of hosting providers including Heroku, Google Cloud Run and Vercel. +The quickest way to deploy a Datasette instance on the internet is to use the ``datasette publish`` command, described in :ref:`publishing`. This can be used to quickly deploy Datasette to a number of hosting providers including Heroku, Google Cloud Run and Vercel. You can deploy Datasette to other hosting providers using the instructions on this page. @@ -109,7 +109,7 @@ If you want to build SQLite files or download them as part of the deployment pro wget https://fivethirtyeight.datasettes.com/fivethirtyeight.db -`simonw/buildpack-datasette-demo `__ is an example GitHub repository showing a simple Datasette configuration that can be deployed to a buildpack-supporting host. +`simonw/buildpack-datasette-demo `__ is an example GitHub repository showing a Datasette configuration that can be deployed to a buildpack-supporting host. .. _deploying_proxy: diff --git a/docs/ecosystem.rst b/docs/ecosystem.rst index 4b80e71e..2ab4224a 100644 --- a/docs/ecosystem.rst +++ b/docs/ecosystem.rst @@ -68,7 +68,7 @@ For example, to create a SQLite database of the `City of Dallas Payment Register Datasette Plugins ================= -Datasette's :ref:`plugin system ` makes it easy to enhance Datasette with additional functionality. +Datasette's :ref:`plugin system ` allows developers to enhance Datasette with additional functionality. datasette-graphql ----------------- diff --git a/docs/internals.rst b/docs/internals.rst index d3d0be8e..92496490 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -119,7 +119,7 @@ For example: content_type="application/xml; charset=utf-8" ) -The easiest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: +The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods: .. code-block:: python diff --git a/docs/metadata.rst b/docs/metadata.rst index 471a52e3..87c81ff6 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -310,7 +310,7 @@ Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`can where neighborhood like '%' || :text || '%' order by neighborhood; title: Search neighborhoods description_html: |- -

    This demonstrates simple LIKE search +

    This demonstrates basic LIKE search The ``metadata.yml`` file is passed to Datasette using the same ``--metadata`` option:: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 6f8d269d..8407a259 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -471,7 +471,7 @@ It can also return a dictionary with the following keys. This format is **deprec ``headers`` - dictionary, optional Extra HTTP headers to be returned in the response. -A simple example of an output renderer callback function: +An example of an output renderer callback function: .. code-block:: python diff --git a/docs/publish.rst b/docs/publish.rst index 45048ce1..a905ac92 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -71,7 +71,7 @@ You can specify a custom app name by passing ``-n my-app-name`` to the publish c Publishing to Vercel -------------------- -`Vercel `__ - previously known as Zeit Now - provides a layer over AWS Lambda to allow for easy, scale-to-zero deployment. You can deploy Datasette instances to Vercel using the `datasette-publish-vercel `__ plugin. +`Vercel `__ - previously known as Zeit Now - provides a layer over AWS Lambda to allow for quick, scale-to-zero deployment. You can deploy Datasette instances to Vercel using the `datasette-publish-vercel `__ plugin. :: @@ -85,7 +85,7 @@ Not every feature is supported: consult the `datasette-publish-vercel README `__ is a `competitively priced `__ Docker-compatible hosting platform that makes it easy to run applications in globally distributed data centers close to your end users. You can deploy Datasette instances to Fly using the `datasette-publish-fly `__ plugin. +`Fly `__ is a `competitively priced `__ Docker-compatible hosting platform that supports running applications in globally distributed data centers close to your end users. You can deploy Datasette instances to Fly using the `datasette-publish-fly `__ plugin. :: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 0ce506cb..ec4c860e 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -64,7 +64,7 @@ If you want to bundle some pre-written SQL queries with your Datasette-hosted database you can do so in two ways. The first is to include SQL views in your database - Datasette will then list those views on your database index page. -The easiest way to create views is with the SQLite command-line interface:: +The quickest way to create views is with the SQLite command-line interface:: $ sqlite3 sf-trees.db SQLite version 3.19.3 2017-06-27 16:48:08 diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index dfcda8a9..60d5056a 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -10,7 +10,7 @@ You can write one-off plugins that apply to just one Datasette instance, or you Writing one-off plugins ----------------------- -The easiest way to write a plugin is to create a ``my_plugin.py`` file and drop it into your ``plugins/`` directory. Here is an example plugin, which adds a new custom SQL function called ``hello_world()`` which takes no arguments and returns the string ``Hello world!``. +The quickest way to start writing a plugin is to create a ``my_plugin.py`` file and drop it into your ``plugins/`` directory. Here is an example plugin, which adds a new custom SQL function called ``hello_world()`` which takes no arguments and returns the string ``Hello world!``. .. code-block:: python @@ -37,7 +37,7 @@ Starting an installable plugin using cookiecutter Plugins that can be installed should be written as Python packages using a ``setup.py`` file. -The easiest way to start writing one an installable plugin is to use the `datasette-plugin `__ cookiecutter template. This creates a new plugin structure for you complete with an example test and GitHub Actions workflows for testing and publishing your plugin. +The quickest way to start writing one an installable plugin is to use the `datasette-plugin `__ cookiecutter template. This creates a new plugin structure for you complete with an example test and GitHub Actions workflows for testing and publishing your plugin. `Install cookiecutter `__ and then run this command to start building a plugin using the template:: From 200284e1a7541af62c7df5467acfb7edd0ee934a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 15 Nov 2020 08:43:13 -0800 Subject: [PATCH 0165/1866] Clarified how --plugin-secret works --- docs/plugins.rst | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 1c0dd588..06e2ec00 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -123,7 +123,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship You can add the ``--plugins-dir=`` option to include any plugins found in that directory. - .. _plugins_configuration: Plugin configuration @@ -131,7 +130,9 @@ Plugin configuration Plugins can have their own configuration, embedded in a :ref:`metadata` file. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. -Here is an example of some plugin configuration for a specific table:: +Here is an example of some plugin configuration for a specific table: + +.. code-block:: json { "databases: { @@ -159,7 +160,9 @@ Secret configuration values Any values embedded in ``metadata.json`` will be visible to anyone who views the ``/-/metadata`` page of your Datasette instance. Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. -**As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so:: +**As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so: + +.. code-block:: json { "plugins": { @@ -171,7 +174,9 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } -**As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this:: +**As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this: + +.. code-block:: json { "plugins": { @@ -190,3 +195,20 @@ If you are publishing your data using the :ref:`datasette publish ` --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ --plugin-secret datasette-auth-github client_secret your_client_secret + +This will set the necessary environment variables and add the following to the deployed ``metadata.json``: + +.. code-block:: json + + { + "plugins": { + "datasette-auth-github": { + "client_id": { + "$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID" + }, + "client_secret": { + "$env": "DATASETTE_AUTH_GITHUB_CLIENT_SECRET" + } + } + } + } From 6fd35be64de221eba4945ca24e8e1678f6142a73 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 15 Nov 2020 08:45:26 -0800 Subject: [PATCH 0166/1866] Fixed invalid JSON in exampl --- docs/plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 06e2ec00..3e756a9e 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -135,7 +135,7 @@ Here is an example of some plugin configuration for a specific table: .. code-block:: json { - "databases: { + "databases": { "sf-trees": { "tables": { "Street_Tree_List": { From 30e64c8d3b3728a86c3ca42a75322cc3feb5b0c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 15 Nov 2020 15:24:22 -0800 Subject: [PATCH 0167/1866] Use f-strings in place of .format() Code transformed like so: pip install flynt flynt . black . --- datasette/app.py | 14 +++--- datasette/blob_renderer.py | 6 +-- datasette/cli.py | 20 +++----- datasette/database.py | 12 ++--- datasette/facets.py | 20 ++++---- datasette/filters.py | 18 +++---- datasette/inspect.py | 2 +- datasette/publish/cloudrun.py | 8 ++- datasette/publish/common.py | 4 +- datasette/publish/heroku.py | 14 ++---- datasette/renderer.py | 4 +- datasette/tracer.py | 4 +- datasette/url_builder.py | 14 +++--- datasette/utils/__init__.py | 91 +++++++++++++++------------------- datasette/utils/asgi.py | 2 +- datasette/utils/testing.py | 6 +-- datasette/views/base.py | 22 ++++---- datasette/views/database.py | 8 ++- datasette/views/special.py | 4 +- datasette/views/table.py | 88 +++++++++++++------------------- tests/fixtures.py | 12 ++--- tests/plugins/my_plugin.py | 16 +++--- tests/test_api.py | 8 +-- tests/test_auth.py | 2 +- tests/test_canned_queries.py | 8 +-- tests/test_cli.py | 2 +- tests/test_docs.py | 6 +-- tests/test_filters.py | 4 +- tests/test_html.py | 41 +++++++-------- tests/test_internals_urls.py | 2 +- tests/test_messages.py | 2 +- tests/test_plugins.py | 12 ++--- tests/test_publish_cloudrun.py | 10 ++-- tests/test_utils.py | 2 +- update-docs-help.py | 2 +- 35 files changed, 213 insertions(+), 277 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1271e52f..b2bdb746 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -231,7 +231,7 @@ class Datasette: is_mutable = path not in self.immutables db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) if db.name in self.databases: - raise Exception("Multiple files with same stem: {}".format(db.name)) + raise Exception(f"Multiple files with same stem: {db.name}") self.add_database(db.name, db) self.cache_headers = cache_headers self.cors = cors @@ -455,9 +455,9 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - conn.execute("SELECT load_extension('{}')".format(extension)) + conn.execute(f"SELECT load_extension('{extension}')") if self.config("cache_size_kb"): - conn.execute("PRAGMA cache_size=-{}".format(self.config("cache_size_kb"))) + conn.execute(f"PRAGMA cache_size=-{self.config('cache_size_kb')}") # pylint: disable=no-member pm.hook.prepare_connection(conn=conn, database=database, datasette=self) @@ -860,7 +860,7 @@ class Datasette: if plugin["static_path"]: add_route( asgi_static(plugin["static_path"]), - "/-/static-plugins/{}/(?P.*)$".format(plugin["name"]), + f"/-/static-plugins/{plugin['name']}/(?P.*)$", ) # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611 add_route( @@ -1156,7 +1156,7 @@ class DatasetteRouter: info = {} message = str(exception) traceback.print_exc() - templates = ["{}.html".format(status), "error.html"] + templates = [f"{status}.html", "error.html"] info.update( { "ok": False, @@ -1234,7 +1234,7 @@ def route_pattern_from_filepath(filepath): re_bits = ["/"] for bit in _curly_re.split(filepath): if _curly_re.match(bit): - re_bits.append("(?P<{}>[^/]*)".format(bit[1:-1])) + re_bits.append(f"(?P<{bit[1:-1]}>[^/]*)") else: re_bits.append(re.escape(bit)) return re.compile("^" + "".join(re_bits) + "$") @@ -1253,7 +1253,7 @@ class DatasetteClient: if not isinstance(path, PrefixedUrlString): path = self.ds.urls.path(path) if path.startswith("/"): - path = "http://localhost{}".format(path) + path = f"http://localhost{path}" return path async def get(self, path, **kwargs): diff --git a/datasette/blob_renderer.py b/datasette/blob_renderer.py index 794b153e..217b3638 100644 --- a/datasette/blob_renderer.py +++ b/datasette/blob_renderer.py @@ -9,10 +9,10 @@ _BLOB_HASH = "_blob_hash" async def render_blob(datasette, database, rows, columns, request, table, view_name): if _BLOB_COLUMN not in request.args: - raise BadRequest("?{}= is required".format(_BLOB_COLUMN)) + raise BadRequest(f"?{_BLOB_COLUMN}= is required") blob_column = request.args[_BLOB_COLUMN] if blob_column not in columns: - raise BadRequest("{} is not a valid column".format(blob_column)) + raise BadRequest(f"{blob_column} is not a valid column") # If ?_blob_hash= provided, use that to select the row - otherwise use first row blob_hash = None @@ -42,7 +42,7 @@ async def render_blob(datasette, database, rows, columns, request, table, view_n filename = "-".join(filename_bits) + ".blob" headers = { "X-Content-Type-Options": "nosniff", - "Content-Disposition": 'attachment; filename="{}"'.format(filename), + "Content-Disposition": f'attachment; filename="{filename}"', } return Response( body=value or b"", diff --git a/datasette/cli.py b/datasette/cli.py index 04d2950b..99075078 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -33,12 +33,12 @@ class Config(click.ParamType): def convert(self, config, param, ctx): if ":" not in config: - self.fail('"{}" should be name:value'.format(config), param, ctx) + self.fail(f'"{config}" should be name:value', param, ctx) return name, value = config.split(":", 1) if name not in DEFAULT_CONFIG: self.fail( - "{} is not a valid option (--help-config to see all)".format(name), + f"{name} is not a valid option (--help-config to see all)", param, ctx, ) @@ -49,13 +49,11 @@ class Config(click.ParamType): try: return name, value_as_boolean(value) except ValueAsBooleanError: - self.fail( - '"{}" should be on/off/true/false/1/0'.format(name), param, ctx - ) + self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) return elif isinstance(default, int): if not value.isdigit(): - self.fail('"{}" should be an integer'.format(name), param, ctx) + self.fail(f'"{name}" should be an integer', param, ctx) return return name, int(value) elif isinstance(default, str): @@ -203,7 +201,7 @@ def package( version_note, secret, port, - **extra_metadata + **extra_metadata, ): "Package specified SQLite files into a new datasette Docker container" if not shutil.which("docker"): @@ -389,7 +387,7 @@ def serve( with formatter.section("Config options"): formatter.write_dl( [ - (option.name, "{} (default={})".format(option.help, option.default)) + (option.name, f"{option.help} (default={option.default})") for option in CONFIG_OPTIONS ] ) @@ -470,7 +468,7 @@ def serve( path = asyncio.get_event_loop().run_until_complete( initial_path_for_datasette(ds) ) - url = "http://{}:{}{}".format(host, port, path) + url = f"http://{host}:{port}{path}" webbrowser.open(url) uvicorn.run( ds.app(), host=host, port=port, log_level="info", lifespan="on", workers=1 @@ -491,7 +489,5 @@ async def check_databases(ds): ) except ConnectionProblem as e: raise click.UsageError( - "Connection to {} failed check: {}".format( - database.path, str(e.args[0]) - ) + f"Connection to {database.path} failed check: {str(e.args[0])}" ) diff --git a/datasette/database.py b/datasette/database.py index a9f39253..ea1424a5 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -57,7 +57,7 @@ class Database: if write: qs = "" return sqlite3.connect( - "file:{}{}".format(self.path, qs), uri=True, check_same_thread=False + f"file:{self.path}{qs}", uri=True, check_same_thread=False ) async def execute_write(self, sql, params=None, block=False): @@ -191,7 +191,7 @@ class Database: try: table_count = ( await self.execute( - "select count(*) from [{}]".format(table), + f"select count(*) from [{table}]", custom_time_limit=limit, ) ).rows[0][0] @@ -362,13 +362,13 @@ class Database: if self.is_memory: tags.append("memory") if self.hash: - tags.append("hash={}".format(self.hash)) + tags.append(f"hash={self.hash}") if self.size is not None: - tags.append("size={}".format(self.size)) + tags.append(f"size={self.size}") tags_str = "" if tags: - tags_str = " ({})".format(", ".join(tags)) - return "".format(self.name, tags_str) + tags_str = f" ({', '.join(tags)})" + return f"" class WriteTask: diff --git a/datasette/facets.py b/datasette/facets.py index 1712db9b..a818a9e9 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -86,7 +86,7 @@ class Facet: self.database = database # For foreign key expansion. Can be None for e.g. canned SQL queries: self.table = table - self.sql = sql or "select * from [{}]".format(table) + self.sql = sql or f"select * from [{table}]" self.params = params or [] self.metadata = metadata # row_count can be None, in which case we calculate it ourselves: @@ -114,7 +114,7 @@ class Facet: # Detect column names using the "limit 0" trick return ( await self.ds.execute( - self.database, "select * from ({}) limit 0".format(sql), params or [] + self.database, f"select * from ({sql}) limit 0", params or [] ) ).columns @@ -123,7 +123,7 @@ class Facet: self.row_count = ( await self.ds.execute( self.database, - "select count(*) from ({})".format(self.sql), + f"select count(*) from ({self.sql})", self.params, ) ).rows[0][0] @@ -371,14 +371,14 @@ class ArrayFacet(Facet): pairs = self.get_querystring_pairs() for row in facet_rows: value = str(row["value"]) - selected = ("{}__arraycontains".format(column), value) in pairs + selected = (f"{column}__arraycontains", value) in pairs if selected: toggle_path = path_with_removed_args( - self.request, {"{}__arraycontains".format(column): value} + self.request, {f"{column}__arraycontains": value} ) else: toggle_path = path_with_added_args( - self.request, {"{}__arraycontains".format(column): value} + self.request, {f"{column}__arraycontains": value} ) facet_results_values.append( { @@ -482,16 +482,14 @@ class DateFacet(Facet): } facet_rows = facet_rows_results.rows[:facet_size] for row in facet_rows: - selected = str(args.get("{}__date".format(column))) == str( - row["value"] - ) + selected = str(args.get(f"{column}__date")) == str(row["value"]) if selected: toggle_path = path_with_removed_args( - self.request, {"{}__date".format(column): str(row["value"])} + self.request, {f"{column}__date": str(row["value"])} ) else: toggle_path = path_with_added_args( - self.request, {"{}__date".format(column): row["value"]} + self.request, {f"{column}__date": row["value"]} ) facet_results_values.append( { diff --git a/datasette/filters.py b/datasette/filters.py index 4891154a..1524b32a 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -43,7 +43,7 @@ class TemplatedFilter(Filter): kwargs = {"c": column} converted = None else: - kwargs = {"c": column, "p": "p{}".format(param_counter), "t": table} + kwargs = {"c": column, "p": f"p{param_counter}", "t": table} return self.sql_template.format(**kwargs), converted def human_clause(self, column, value): @@ -69,12 +69,12 @@ class InFilter(Filter): def where_clause(self, table, column, value, param_counter): values = self.split_value(value) - params = [":p{}".format(param_counter + i) for i in range(len(values))] - sql = "{} in ({})".format(escape_sqlite(column), ", ".join(params)) + params = [f":p{param_counter + i}" for i in range(len(values))] + sql = f"{escape_sqlite(column)} in ({', '.join(params)})" return sql, values def human_clause(self, column, value): - return "{} in {}".format(column, json.dumps(self.split_value(value))) + return f"{column} in {json.dumps(self.split_value(value))}" class NotInFilter(InFilter): @@ -83,12 +83,12 @@ class NotInFilter(InFilter): def where_clause(self, table, column, value, param_counter): values = self.split_value(value) - params = [":p{}".format(param_counter + i) for i in range(len(values))] - sql = "{} not in ({})".format(escape_sqlite(column), ", ".join(params)) + params = [f":p{param_counter + i}" for i in range(len(values))] + sql = f"{escape_sqlite(column)} not in ({', '.join(params)})" return sql, values def human_clause(self, column, value): - return "{} not in {}".format(column, json.dumps(self.split_value(value))) + return f"{column} not in {json.dumps(self.split_value(value))}" class Filters: @@ -221,7 +221,7 @@ class Filters: s = " and ".join(and_bits) if not s: return "" - return "where {}".format(s) + return f"where {s}" def selections(self): "Yields (column, lookup, value) tuples" @@ -265,7 +265,7 @@ class Filters: if not isinstance(param, list): param = [param] for individual_param in param: - param_id = "p{}".format(i) + param_id = f"p{i}" params[param_id] = individual_param i += 1 return sql_bits, params diff --git a/datasette/inspect.py b/datasette/inspect.py index 2324c02c..4d538e5f 100644 --- a/datasette/inspect.py +++ b/datasette/inspect.py @@ -47,7 +47,7 @@ def inspect_tables(conn, database_metadata): try: count = conn.execute( - "select count(*) from {}".format(escape_sqlite(table)) + f"select count(*) from {escape_sqlite(table)}" ).fetchone()[0] except sqlite3.OperationalError: # This can happen when running against a FTS virtual table diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 8f99dc2e..54f55fcb 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -100,9 +100,7 @@ def publish_subcommand(publish): extra_metadata["plugins"] = {} for plugin_name, plugin_setting, setting_value in plugin_secret: environment_variable = ( - "{}_{}".format(plugin_name, plugin_setting) - .upper() - .replace("-", "_") + f"{plugin_name}_{plugin_setting}".upper().replace("-", "_") ) environment_variables[environment_variable] = setting_value extra_metadata["plugins"].setdefault(plugin_name, {})[ @@ -133,8 +131,8 @@ def publish_subcommand(publish): print(open("Dockerfile").read()) print("\n====================\n") - image_id = "gcr.io/{project}/{name}".format(project=project, name=name) - check_call("gcloud builds submit --tag {}".format(image_id), shell=True) + image_id = f"gcr.io/{project}/{name}" + check_call(f"gcloud builds submit --tag {image_id}", shell=True) check_call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, service, " --memory {}".format(memory) if memory else "" diff --git a/datasette/publish/common.py b/datasette/publish/common.py index 49a4798e..b6570290 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -85,9 +85,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link): err=True, ) click.echo( - "Follow the instructions at {install_link}".format( - install_link=install_link - ), + f"Follow the instructions at {install_link}", err=True, ) sys.exit(1) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 24305de5..c772b476 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -83,9 +83,7 @@ def publish_subcommand(publish): extra_metadata["plugins"] = {} for plugin_name, plugin_setting, setting_value in plugin_secret: environment_variable = ( - "{}_{}".format(plugin_name, plugin_setting) - .upper() - .replace("-", "_") + f"{plugin_name}_{plugin_setting}".upper().replace("-", "_") ) environment_variables[environment_variable] = setting_value extra_metadata["plugins"].setdefault(plugin_name, {})[ @@ -129,9 +127,7 @@ def publish_subcommand(publish): app_name = json.loads(create_output)["name"] for key, value in environment_variables.items(): - call( - ["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)] - ) + call(["heroku", "config:set", "-a", app_name, f"{key}={value}"]) tar_option = [] if tar: tar_option = ["--tar", tar] @@ -181,9 +177,7 @@ def temporary_heroku_directory( if branch: install = [ - "https://github.com/simonw/datasette/archive/{branch}.zip".format( - branch=branch - ) + f"https://github.com/simonw/datasette/archive/{branch}.zip" ] + list(install) else: install = ["datasette"] + list(install) @@ -216,7 +210,7 @@ def temporary_heroku_directory( link_or_copy_directory( os.path.join(saved_cwd, path), os.path.join(tmp.name, mount_point) ) - extras.extend(["--static", "{}:{}".format(mount_point, mount_point)]) + extras.extend(["--static", f"{mount_point}:{mount_point}"]) quoted_files = " ".join( ["-i {}".format(shlex.quote(file_name)) for file_name in file_names] diff --git a/datasette/renderer.py b/datasette/renderer.py index bcde8516..d779b44f 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -82,7 +82,7 @@ def json_renderer(args, data, view_name): status_code = 400 data = { "ok": False, - "error": "Invalid _shape: {}".format(shape), + "error": f"Invalid _shape: {shape}", "status": 400, "title": None, } @@ -96,7 +96,7 @@ def json_renderer(args, data, view_name): content_type = "application/json; charset=utf-8" headers = {} if next_url: - headers["link"] = '<{}>; rel="next"'.format(next_url) + headers["link"] = f'<{next_url}>; rel="next"' return Response( body, status=status_code, headers=headers, content_type=content_type ) diff --git a/datasette/tracer.py b/datasette/tracer.py index a638b140..8f666767 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -28,7 +28,7 @@ def get_task_id(): def trace(type, **kwargs): assert not TRACE_RESERVED_KEYS.intersection( kwargs.keys() - ), ".trace() keyword parameters cannot include {}".format(TRACE_RESERVED_KEYS) + ), f".trace() keyword parameters cannot include {TRACE_RESERVED_KEYS}" task_id = get_task_id() if task_id is None: yield @@ -124,7 +124,7 @@ class AsgiTracer: content_type = "" if "text/html" in content_type and b"" in accumulated_body: extra = json.dumps(trace_info, indent=2) - extra_html = "

    {}
    ".format(extra).encode("utf8") + extra_html = f"
    {extra}
    ".encode("utf8") accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): data = json.loads(accumulated_body.decode("utf8")) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index bcc4f39d..697f60ae 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -19,10 +19,10 @@ class Urls: return self.path("", format=format) def static(self, path): - return self.path("-/static/{}".format(path)) + return self.path(f"-/static/{path}") def static_plugins(self, plugin, path): - return self.path("-/static-plugins/{}/{}".format(plugin, path)) + return self.path(f"-/static-plugins/{plugin}/{path}") def logout(self): return self.path("-/logout") @@ -30,27 +30,25 @@ class Urls: def database(self, database, format=None): db = self.ds.databases[database] if self.ds.config("hash_urls") and db.hash: - path = self.path( - "{}-{}".format(database, db.hash[:HASH_LENGTH]), format=format - ) + path = self.path(f"{database}-{db.hash[:HASH_LENGTH]}", format=format) else: path = self.path(database, format=format) return path def table(self, database, table, format=None): - path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(table)) + path = f"{self.database(database)}/{urllib.parse.quote_plus(table)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) def query(self, database, query, format=None): - path = "{}/{}".format(self.database(database), urllib.parse.quote_plus(query)) + path = f"{self.database(database)}/{urllib.parse.quote_plus(query)}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) def row(self, database, table, row_path, format=None): - path = "{}/{}".format(self.table(database, table), row_path) + path = f"{self.table(database, table)}/{row_path}" if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index a7d96401..02b59b2b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -115,13 +115,10 @@ def compound_keys_after_sql(pks, start_index=0): last = pks_left[-1] rest = pks_left[:-1] and_clauses = [ - "{} = :p{}".format(escape_sqlite(pk), (i + start_index)) - for i, pk in enumerate(rest) + f"{escape_sqlite(pk)} = :p{i + start_index}" for i, pk in enumerate(rest) ] - and_clauses.append( - "{} > :p{}".format(escape_sqlite(last), (len(rest) + start_index)) - ) - or_clauses.append("({})".format(" and ".join(and_clauses))) + and_clauses.append(f"{escape_sqlite(last)} > :p{len(rest) + start_index}") + or_clauses.append(f"({' and '.join(and_clauses)})") pks_left.pop() or_clauses.reverse() return "({})".format("\n or\n".join(or_clauses)) @@ -195,7 +192,7 @@ allowed_pragmas = ( ) disallawed_sql_res = [ ( - re.compile("pragma(?!_({}))".format("|".join(allowed_pragmas))), + re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"), "Statement may not contain PRAGMA", ) ] @@ -215,7 +212,7 @@ def validate_sql_select(sql): def append_querystring(url, querystring): op = "&" if ("?" in url) else "?" - return "{}{}{}".format(url, op, querystring) + return f"{url}{op}{querystring}" def path_with_added_args(request, args, path=None): @@ -230,7 +227,7 @@ def path_with_added_args(request, args, path=None): current.extend([(key, value) for key, value in args if value is not None]) query_string = urllib.parse.urlencode(current) if query_string: - query_string = "?{}".format(query_string) + query_string = f"?{query_string}" return path + query_string @@ -259,7 +256,7 @@ def path_with_removed_args(request, args, path=None): current.append((key, value)) query_string = urllib.parse.urlencode(current) if query_string: - query_string = "?{}".format(query_string) + query_string = f"?{query_string}" return path + query_string @@ -275,7 +272,7 @@ def path_with_replaced_args(request, args, path=None): current.extend([p for p in args if p[1] is not None]) query_string = urllib.parse.urlencode(current) if query_string: - query_string = "?{}".format(query_string) + query_string = f"?{query_string}" return path + query_string @@ -285,7 +282,7 @@ _boring_keyword_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") def escape_css_string(s): return _css_re.sub( - lambda m: "\\" + ("{:X}".format(ord(m.group())).zfill(6)), + lambda m: "\\" + (f"{ord(m.group()):X}".zfill(6)), s.replace("\r\n", "\n"), ) @@ -294,7 +291,7 @@ def escape_sqlite(s): if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s else: - return "[{}]".format(s) + return f"[{s}]" def make_dockerfile( @@ -319,27 +316,27 @@ def make_dockerfile( cmd.extend(["-i", filename]) cmd.extend(["--cors", "--inspect-file", "inspect-data.json"]) if metadata_file: - cmd.extend(["--metadata", "{}".format(metadata_file)]) + cmd.extend(["--metadata", f"{metadata_file}"]) if template_dir: cmd.extend(["--template-dir", "templates/"]) if plugins_dir: cmd.extend(["--plugins-dir", "plugins/"]) if version_note: - cmd.extend(["--version-note", "{}".format(version_note)]) + cmd.extend(["--version-note", f"{version_note}"]) if static: for mount_point, _ in static: - cmd.extend(["--static", "{}:{}".format(mount_point, mount_point)]) + cmd.extend(["--static", f"{mount_point}:{mount_point}"]) if extra_options: for opt in extra_options.split(): - cmd.append("{}".format(opt)) + cmd.append(f"{opt}") cmd = [shlex.quote(part) for part in cmd] # port attribute is a (fixed) env variable and should not be quoted cmd.extend(["--port", "$PORT"]) cmd = " ".join(cmd) if branch: - install = [ - "https://github.com/simonw/datasette/archive/{}.zip".format(branch) - ] + list(install) + install = [f"https://github.com/simonw/datasette/archive/{branch}.zip"] + list( + install + ) else: install = ["datasette"] + list(install) @@ -449,7 +446,7 @@ def detect_primary_keys(conn, table): " Figure out primary keys for a table. " table_info_rows = [ row - for row in conn.execute('PRAGMA table_info("{}")'.format(table)).fetchall() + for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() if row[-1] ] table_info_rows.sort(key=lambda row: row[-1]) @@ -457,7 +454,7 @@ def detect_primary_keys(conn, table): def get_outbound_foreign_keys(conn, table): - infos = conn.execute("PRAGMA foreign_key_list([{}])".format(table)).fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() fks = [] for info in infos: if info is not None: @@ -476,7 +473,7 @@ def get_all_foreign_keys(conn): for table in tables: table_to_foreign_keys[table] = {"incoming": [], "outgoing": []} for table in tables: - infos = conn.execute("PRAGMA foreign_key_list([{}])".format(table)).fetchall() + infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() for info in infos: if info is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = info @@ -544,9 +541,7 @@ def table_columns(conn, table): def table_column_details(conn, table): return [ Column(*r) - for r in conn.execute( - "PRAGMA table_info({});".format(escape_sqlite(table)) - ).fetchall() + for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall() ] @@ -562,9 +557,7 @@ def filters_should_redirect(special_args): if "__" in filter_op: filter_op, filter_value = filter_op.split("__", 1) if filter_column: - redirect_params.append( - ("{}__{}".format(filter_column, filter_op), filter_value) - ) + redirect_params.append((f"{filter_column}__{filter_op}", filter_value)) for key in ("_filter_column", "_filter_op", "_filter_value"): if key in special_args: redirect_params.append((key, None)) @@ -573,17 +566,17 @@ def filters_should_redirect(special_args): for column_key in column_keys: number = column_key.split("_")[-1] column = special_args[column_key] - op = special_args.get("_filter_op_{}".format(number)) or "exact" - value = special_args.get("_filter_value_{}".format(number)) or "" + op = special_args.get(f"_filter_op_{number}") or "exact" + value = special_args.get(f"_filter_value_{number}") or "" if "__" in op: op, value = op.split("__", 1) if column: - redirect_params.append(("{}__{}".format(column, op), value)) + redirect_params.append((f"{column}__{op}", value)) redirect_params.extend( [ - ("_filter_column_{}".format(number), None), - ("_filter_op_{}".format(number), None), - ("_filter_value_{}".format(number), None), + (f"_filter_column_{number}", None), + (f"_filter_op_{number}", None), + (f"_filter_value_{number}", None), ] ) return redirect_params @@ -672,7 +665,7 @@ async def resolve_table_and_format( # Check if table ends with a known format formats = list(allowed_formats) + ["csv", "jsono"] for _format in formats: - if table_and_format.endswith(".{}".format(_format)): + if table_and_format.endswith(f".{_format}"): table = table_and_format[: -(len(_format) + 1)] return table, _format return table_and_format, None @@ -683,20 +676,20 @@ def path_with_format( ): qs = extra_qs or {} path = request.path if request else path - if replace_format and path.endswith(".{}".format(replace_format)): + if replace_format and path.endswith(f".{replace_format}"): path = path[: -(1 + len(replace_format))] if "." in path: qs["_format"] = format else: - path = "{}.{}".format(path, format) + path = f"{path}.{format}" if qs: extra = urllib.parse.urlencode(sorted(qs.items())) if request and request.query_string: - path = "{}?{}&{}".format(path, request.query_string, extra) + path = f"{path}?{request.query_string}&{extra}" else: - path = "{}?{}".format(path, extra) + path = f"{path}?{extra}" elif request and request.query_string: - path = "{}?{}".format(path, request.query_string) + path = f"{path}?{request.query_string}" return path @@ -742,9 +735,7 @@ class LimitedWriter: async def write(self, bytes): self.bytes_count += len(bytes) if self.limit_bytes and (self.bytes_count > self.limit_bytes): - raise WriteLimitExceeded( - "CSV contains more than {} bytes".format(self.limit_bytes) - ) + raise WriteLimitExceeded(f"CSV contains more than {self.limit_bytes} bytes") await self.writer.write(bytes) @@ -763,14 +754,14 @@ class StaticMount(click.ParamType): def convert(self, value, param, ctx): if ":" not in value: self.fail( - '"{}" should be of format mountpoint:directory'.format(value), + f'"{value}" should be of format mountpoint:directory', param, ctx, ) path, dirpath = value.split(":", 1) dirpath = os.path.abspath(dirpath) if not os.path.exists(dirpath) or not os.path.isdir(dirpath): - self.fail("%s is not a valid directory path" % value, param, ctx) + self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath @@ -781,9 +772,9 @@ def format_bytes(bytes): break current = current / 1024 if unit == "bytes": - return "{} {}".format(int(current), unit) + return f"{int(current)} {unit}" else: - return "{:.1f} {}".format(current, unit) + return f"{current:.1f} {unit}" _escape_fts_re = re.compile(r'\s+|(".*?")') @@ -820,7 +811,7 @@ class MultiParams: self._data = new_data def __repr__(self): - return "".format(self._data) + return f"" def __contains__(self, key): return key in self._data @@ -867,7 +858,7 @@ def check_connection(conn): for table in tables: try: conn.execute( - "PRAGMA table_info({});".format(escape_sqlite(table)), + f"PRAGMA table_info({escape_sqlite(table)});", ) except sqlite3.OperationalError as e: if e.args[0] == "no such module: VirtualSpatialIndex": diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index e4c8ce5c..ce78a597 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -260,7 +260,7 @@ async def asgi_send_file( ): headers = headers or {} if filename: - headers["content-disposition"] = 'attachment; filename="{}"'.format(filename) + headers["content-disposition"] = f'attachment; filename="{filename}"' first = True headers["content-length"] = str((await aiofiles.os.stat(str(filepath))).st_size) async with aiofiles.open(str(filepath), mode="rb") as fp: diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 8a8810e7..bcbc1c7a 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -32,7 +32,7 @@ class TestResponse: return any( h for h in self.httpx_response.headers.get_list("set-cookie") - if h.startswith('{}="";'.format(cookie)) + if h.startswith(f'{cookie}="";') ) @property @@ -125,9 +125,7 @@ class TestClient: if allow_redirects and response.status in (301, 302): assert ( redirect_count < self.max_redirects - ), "Redirected {} times, max_redirects={}".format( - redirect_count, self.max_redirects - ) + ), f"Redirected {redirect_count} times, max_redirects={self.max_redirects}" location = response.headers["Location"] return await self._request( location, allow_redirects=True, redirect_count=redirect_count + 1 diff --git a/datasette/views/base.py b/datasette/views/base.py index 430489c1..b3a54bcc 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -125,9 +125,7 @@ class BaseView: **{ "database_color": self.database_color, "select_templates": [ - "{}{}".format( - "*" if template_name == template.name else "", template_name - ) + f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], }, @@ -165,11 +163,11 @@ class DataView(BaseView): def redirect(self, request, path, forward_querystring=True, remove_args=None): if request.query_string and "?" not in path and forward_querystring: - path = "{}?{}".format(path, request.query_string) + path = f"{path}?{request.query_string}" if remove_args: path = path_with_removed_args(request, remove_args, path=path) r = Response.redirect(path) - r.headers["Link"] = "<{}>; rel=preload".format(path) + r.headers["Link"] = f"<{path}>; rel=preload" if self.ds.cors: r.headers["Access-Control-Allow-Origin"] = "*" return r @@ -184,7 +182,7 @@ class DataView(BaseView): # No matching DB found, maybe it's a name-hash? name_bit, hash_bit = db_name.rsplit("-", 1) if name_bit not in self.ds.databases: - raise NotFound("Database not found: {}".format(name)) + raise NotFound(f"Database not found: {name}") else: name = name_bit hash = hash_bit @@ -194,7 +192,7 @@ class DataView(BaseView): try: db = self.ds.databases[name] except KeyError: - raise NotFound("Database not found: {}".format(name)) + raise NotFound(f"Database not found: {name}") # Verify the hash expected = "000" @@ -217,11 +215,11 @@ class DataView(BaseView): ) kwargs["table"] = table if _format: - kwargs["as_format"] = ".{}".format(_format) + kwargs["as_format"] = f".{_format}" elif kwargs.get("table"): kwargs["table"] = urllib.parse.unquote_plus(kwargs["table"]) - should_redirect = self.ds.urls.path("{}-{}".format(name, expected)) + should_redirect = self.ds.urls.path(f"{name}-{expected}") if kwargs.get("table"): should_redirect += "/" + urllib.parse.quote_plus(kwargs["table"]) if kwargs.get("pk_path"): @@ -294,7 +292,7 @@ class DataView(BaseView): for column in data["columns"]: headings.append(column) if column in expanded_columns: - headings.append("{}_label".format(column)) + headings.append(f"{column}_label") async def stream_fn(r): nonlocal data @@ -505,7 +503,7 @@ class DataView(BaseView): elif isinstance(result, Response): r = result else: - assert False, "{} should be dict or Response".format(result) + assert False, f"{result} should be dict or Response" else: extras = {} if callable(extra_template_data): @@ -581,7 +579,7 @@ class DataView(BaseView): if ttl == 0: ttl_header = "no-cache" else: - ttl_header = "max-age={}".format(ttl) + ttl_header = f"max-age={ttl}" response.headers["Cache-Control"] = ttl_header response.headers["Referrer-Policy"] = "no-referrer" if self.ds.cors: diff --git a/datasette/views/database.py b/datasette/views/database.py index 9a7b96fd..d4ed8570 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -140,7 +140,7 @@ class DatabaseView(DataView): and not db.is_mutable and database != ":memory:", }, - ("database-{}.html".format(to_css_class(database)), "database.html"), + (f"database-{to_css_class(database)}.html", "database.html"), ) @@ -233,7 +233,7 @@ class QueryView(DataView): if _size: extra_args["page_size"] = _size - templates = ["query-{}.html".format(to_css_class(database)), "query.html"] + templates = [f"query-{to_css_class(database)}.html", "query.html"] # Execute query - as write or as read if write: @@ -324,9 +324,7 @@ class QueryView(DataView): if canned_query: templates.insert( 0, - "query-{}-{}.html".format( - to_css_class(database), to_css_class(canned_query) - ), + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", ) allow_execute_sql = await self.ds.permission_allowed( diff --git a/datasette/views/special.py b/datasette/views/special.py index 397dbc8c..9750dd06 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -111,13 +111,13 @@ class AllowDebugView(BaseView): actor = json.loads(actor_input) actor_input = json.dumps(actor, indent=4) except json.decoder.JSONDecodeError as ex: - errors.append("Actor JSON error: {}".format(ex)) + errors.append(f"Actor JSON error: {ex}") allow_input = request.args.get("allow") or '{"id": "*"}' try: allow = json.loads(allow_input) allow_input = json.dumps(allow, indent=4) except json.decoder.JSONDecodeError as ex: - errors.append("Allow JSON error: {}".format(ex)) + errors.append(f"Allow JSON error: {ex}") result = None if not errors: diff --git a/datasette/views/table.py b/datasette/views/table.py index 9ed45df1..09c2d740 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -212,13 +212,11 @@ class RowTableShared(DataView): # representation, which we have to round off to avoid ugliness. In the vast # majority of cases this rounding will be inconsequential. I hope. value = round(value.to_compact(), 6) - display_value = jinja2.Markup( - "{:~P}".format(value).replace(" ", " ") - ) + display_value = jinja2.Markup(f"{value:~P}".replace(" ", " ")) else: display_value = str(value) if truncate_cells and len(display_value) > truncate_cells: - display_value = display_value[:truncate_cells] + u"\u2026" + display_value = display_value[:truncate_cells] + "\u2026" cells.append( { @@ -307,7 +305,7 @@ class TableView(RowTableShared): is_view = bool(await db.get_view_definition(table)) table_exists = bool(await db.table_exists(table)) if not is_view and not table_exists: - raise NotFound("Table not found: {}".format(table)) + raise NotFound(f"Table not found: {table}") await self.check_permissions( request, @@ -330,7 +328,7 @@ class TableView(RowTableShared): use_rowid = not pks and not is_view if use_rowid: - select = "rowid, {}".format(select_columns) + select = f"rowid, {select_columns}" order_by = "rowid" order_by_pks = "rowid" else: @@ -424,7 +422,7 @@ class TableView(RowTableShared): raise DatasetteError( "Invalid _through - could not find corresponding foreign key" ) - param = "p{}".format(len(params)) + param = f"p{len(params)}" where_clauses.append( "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format( through_table=escape_sqlite(through_table), @@ -436,7 +434,7 @@ class TableView(RowTableShared): ) params[param] = value extra_human_descriptions.append( - '{}.{} = "{}"'.format(through_table, other_column, value) + f'{through_table}.{other_column} = "{value}"' ) # _search support: @@ -462,7 +460,7 @@ class TableView(RowTableShared): else "escape_fts(:search)", ) ) - extra_human_descriptions.append('search matches "{}"'.format(search)) + extra_human_descriptions.append(f'search matches "{search}"') params["search"] = search else: # More complex: search against specific columns @@ -481,11 +479,9 @@ class TableView(RowTableShared): ) ) extra_human_descriptions.append( - 'search column "{}" matches "{}"'.format( - search_col, search_text - ) + f'search column "{search_col}" matches "{search_text}"' ) - params["search_{}".format(i)] = search_text + params[f"search_{i}"] = search_text sortable_columns = set() @@ -506,15 +502,15 @@ class TableView(RowTableShared): if sort: if sort not in sortable_columns: - raise DatasetteError("Cannot sort table by {}".format(sort)) + raise DatasetteError(f"Cannot sort table by {sort}") order_by = escape_sqlite(sort) if sort_desc: if sort_desc not in sortable_columns: - raise DatasetteError("Cannot sort table by {}".format(sort_desc)) + raise DatasetteError(f"Cannot sort table by {sort_desc}") - order_by = "{} desc".format(escape_sqlite(sort_desc)) + order_by = f"{escape_sqlite(sort_desc)} desc" from_sql = "from {table_name} {where}".format( table_name=escape_sqlite(table), @@ -525,14 +521,14 @@ class TableView(RowTableShared): # Copy of params so we can mutate them later: from_sql_params = dict(**params) - count_sql = "select count(*) {}".format(from_sql) + count_sql = f"select count(*) {from_sql}" _next = _next or special_args.get("_next") offset = "" if _next: if is_view: # _next is an offset - offset = " offset {}".format(int(_next)) + offset = f" offset {int(_next)}" else: components = urlsafe_components(_next) # If a sort order is applied, the first of these is the sort value @@ -546,8 +542,8 @@ class TableView(RowTableShared): # Figure out the SQL for next-based-on-primary-key first next_by_pk_clauses = [] if use_rowid: - next_by_pk_clauses.append("rowid > :p{}".format(len(params))) - params["p{}".format(len(params))] = components[0] + next_by_pk_clauses.append(f"rowid > :p{len(params)}") + params[f"p{len(params)}"] = components[0] else: # Apply the tie-breaker based on primary keys if len(components) == len(pks): @@ -556,7 +552,7 @@ class TableView(RowTableShared): compound_keys_after_sql(pks, param_len) ) for i, pk_value in enumerate(components): - params["p{}".format(param_len + i)] = pk_value + params[f"p{param_len + i}"] = pk_value # Now add the sort SQL, which may incorporate next_by_pk_clauses if sort or sort_desc: @@ -590,17 +586,17 @@ class TableView(RowTableShared): next_clauses=" and ".join(next_by_pk_clauses), ) ) - params["p{}".format(len(params))] = sort_value - order_by = "{}, {}".format(order_by, order_by_pks) + params[f"p{len(params)}"] = sort_value + order_by = f"{order_by}, {order_by_pks}" else: where_clauses.extend(next_by_pk_clauses) where_clause = "" if where_clauses: - where_clause = "where {} ".format(" and ".join(where_clauses)) + where_clause = f"where {' and '.join(where_clauses)} " if order_by: - order_by = "order by {} ".format(order_by) + order_by = f"order by {order_by} " extra_args = {} # Handle ?_size=500 @@ -617,9 +613,7 @@ class TableView(RowTableShared): raise BadRequest("_size must be a positive integer") if page_size > self.ds.max_returned_rows: - raise BadRequest( - "_size must be <= {}".format(self.ds.max_returned_rows) - ) + raise BadRequest(f"_size must be <= {self.ds.max_returned_rows}") extra_args["page_size"] = page_size else: @@ -631,9 +625,7 @@ class TableView(RowTableShared): where=where_clause, order_by=order_by, ) - sql = "{sql_no_limit} limit {limit}{offset}".format( - sql_no_limit=sql_no_limit.rstrip(), limit=page_size + 1, offset=offset - ) + sql = f"{sql_no_limit.rstrip()} limit {page_size + 1}{offset}" if request.args.get("_timelimit"): extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) @@ -645,7 +637,7 @@ class TableView(RowTableShared): if ( not db.is_mutable and self.ds.inspect_data - and count_sql == "select count(*) from {} ".format(table) + and count_sql == f"select count(*) from {table} " ): try: filtered_table_rows_count = self.ds.inspect_data[database]["tables"][ @@ -763,7 +755,7 @@ class TableView(RowTableShared): prefix = "$null" else: prefix = urllib.parse.quote_plus(str(prefix)) - next_value = "{},{}".format(prefix, next_value) + next_value = f"{prefix},{next_value}" added_args = {"_next": next_value} if sort: added_args["_sort"] = sort @@ -879,12 +871,8 @@ class TableView(RowTableShared): "sort_desc": sort_desc, "disable_sort": is_view, "custom_table_templates": [ - "_table-{}-{}.html".format( - to_css_class(database), to_css_class(table) - ), - "_table-table-{}-{}.html".format( - to_css_class(database), to_css_class(table) - ), + f"_table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-table-{to_css_class(database)}-{to_css_class(table)}.html", "_table.html", ], "metadata": metadata, @@ -918,7 +906,7 @@ class TableView(RowTableShared): }, extra_template, ( - "table-{}-{}.html".format(to_css_class(database), to_css_class(table)), + f"table-{to_css_class(database)}-{to_css_class(table)}.html", "table.html", ), ) @@ -931,13 +919,11 @@ async def _sql_params_pks(db, table, pk_values): if use_rowid: select = "rowid, *" pks = ["rowid"] - wheres = ['"{}"=:p{}'.format(pk, i) for i, pk in enumerate(pks)] - sql = "select {} from {} where {}".format( - select, escape_sqlite(table), " AND ".join(wheres) - ) + wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)] + sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}" params = {} for i, pk_value in enumerate(pk_values): - params["p{}".format(i)] = pk_value + params[f"p{i}"] = pk_value return sql, params, pks @@ -960,7 +946,7 @@ class RowView(RowTableShared): columns = [r[0] for r in results.description] rows = list(results.rows) if not rows: - raise NotFound("Record not found: {}".format(pk_values)) + raise NotFound(f"Record not found: {pk_values}") async def template_data(): display_columns, display_rows = await self.display_columns_and_rows( @@ -981,12 +967,8 @@ class RowView(RowTableShared): "display_columns": display_columns, "display_rows": display_rows, "custom_table_templates": [ - "_table-{}-{}.html".format( - to_css_class(database), to_css_class(table) - ), - "_table-row-{}-{}.html".format( - to_css_class(database), to_css_class(table) - ), + f"_table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", "_table.html", ], "metadata": (self.ds.metadata("databases") or {}) @@ -1014,7 +996,7 @@ class RowView(RowTableShared): data, template_data, ( - "row-{}-{}.html".format(to_css_class(database), to_css_class(table)), + f"row-{to_css_class(database)}-{to_css_class(table)}.html", "row.html", ), ) diff --git a/tests/fixtures.py b/tests/fixtures.py index 183b8ca4..3abca821 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -247,7 +247,7 @@ def generate_compound_rows(num): for a, b, c in itertools.islice( itertools.product(string.ascii_lowercase, repeat=3), num ): - yield a, b, c, "{}-{}-{}".format(a, b, c) + yield a, b, c, f"{a}-{b}-{c}" def generate_sortable_rows(num): @@ -258,7 +258,7 @@ def generate_sortable_rows(num): yield { "pk1": a, "pk2": b, - "content": "{}-{}".format(a, b), + "content": f"{a}-{b}", "sortable": rand.randint(-100, 100), "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), @@ -742,7 +742,7 @@ def cli(db_filename, metadata, plugins_path, recreate): if pathlib.Path(db_filename).exists(): if not recreate: raise click.ClickException( - "{} already exists, use --recreate to reset it".format(db_filename) + f"{db_filename} already exists, use --recreate to reset it" ) else: pathlib.Path(db_filename).unlink() @@ -751,10 +751,10 @@ def cli(db_filename, metadata, plugins_path, recreate): for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) - print("Test tables written to {}".format(db_filename)) + print(f"Test tables written to {db_filename}") if metadata: open(metadata, "w").write(json.dumps(METADATA, indent=4)) - print("- metadata written to {}".format(metadata)) + print(f"- metadata written to {metadata}") if plugins_path: path = pathlib.Path(plugins_path) if not path.exists(): @@ -763,7 +763,7 @@ def cli(db_filename, metadata, plugins_path, recreate): for filepath in test_plugins.glob("*.py"): newpath = path / filepath.name newpath.write_text(filepath.open().read()) - print(" Wrote plugin: {}".format(newpath)) + print(f" Wrote plugin: {newpath}") if __name__ == "__main__": diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 4ac3953b..3f5ec832 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -130,7 +130,7 @@ def extra_template_vars( @hookimpl def prepare_jinja2_environment(env): - env.filters["format_numeric"] = lambda s: "{:,.0f}".format(float(s)) + env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" @hookimpl @@ -207,7 +207,7 @@ def register_routes(): async def two(request): name = request.url_vars["name"] greeting = request.args.get("greeting") - return Response.text("{} {}".format(greeting, name)) + return Response.text(f"{greeting} {name}") async def three(scope, send): await asgi_send_json( @@ -281,11 +281,7 @@ def startup(datasette): @hookimpl def canned_queries(datasette, database, actor): - return { - "from_hook": "select 1, '{}' as actor_id".format( - actor["id"] if actor else "null" - ) - } + return {"from_hook": f"select 1, '{actor['id'] if actor else 'null'}' as actor_id"} @hookimpl @@ -329,9 +325,9 @@ def table_actions(datasette, database, table, actor): return [ { "href": datasette.urls.instance(), - "label": "Database: {}".format(database), + "label": f"Database: {database}", }, - {"href": datasette.urls.instance(), "label": "Table: {}".format(table)}, + {"href": datasette.urls.instance(), "label": f"Table: {table}"}, ] @@ -341,6 +337,6 @@ def database_actions(datasette, database, actor): return [ { "href": datasette.urls.instance(), - "label": "Database: {}".format(database), + "label": f"Database: {database}", } ] diff --git a/tests/test_api.py b/tests/test_api.py index 93097574..3d48d350 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -918,7 +918,7 @@ def test_paginate_compound_keys_with_extra_filters(app_client): ], ) def test_sortable(app_client, query_string, sort_key, human_description_en): - path = "/fixtures/sortable.json?_shape=objects&{}".format(query_string) + path = f"/fixtures/sortable.json?_shape=objects&{query_string}" fetched = [] page = 0 while path: @@ -969,8 +969,8 @@ def test_sortable_columns_metadata(app_client): assert "Cannot sort table by content" == response.json["error"] # no_primary_key has ALL sort options disabled for column in ("content", "a", "b", "c"): - response = app_client.get("/fixtures/sortable.json?_sort={}".format(column)) - assert "Cannot sort table by {}".format(column) == response.json["error"] + response = app_client.get(f"/fixtures/sortable.json?_sort={column}") + assert f"Cannot sort table by {column}" == response.json["error"] @pytest.mark.parametrize( @@ -1877,7 +1877,7 @@ def test_binary_data_in_json(app_client, path, expected_json, expected_text): ], ) def test_paginate_using_link_header(app_client, qs): - path = "/fixtures/compound_three_primary_keys.json{}".format(qs) + path = f"/fixtures/compound_three_primary_keys.json{qs}" num_pages = 0 while path: response = app_client.get(path) diff --git a/tests/test_auth.py b/tests/test_auth.py index 34138aa6..5f3985db 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -7,7 +7,7 @@ import time def test_auth_token(app_client): "The /-/auth-token endpoint sets the correct cookie" assert app_client.ds._root_token is not None - path = "/-/auth-token?token={}".format(app_client.ds._root_token) + path = f"/-/auth-token?token={app_client.ds._root_token}" response = app_client.get( path, allow_redirects=False, diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 9620c693..65f23cc7 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -82,7 +82,7 @@ def test_insert(canned_write_client): def test_canned_query_form_csrf_hidden_field( canned_write_client, query_name, expect_csrf_hidden_field ): - response = canned_write_client.get("/data/{}".format(query_name)) + response = canned_write_client.get(f"/data/{query_name}") html = response.text fragment = '".format(expected_considered) in response.text - ) + assert f"" in response.text def test_table_html_simple_primary_key(app_client): @@ -607,9 +605,7 @@ def test_table_html_simple_primary_key(app_client): for expected_col, th in zip(("content",), ths[1:]): a = th.find("a") assert expected_col == a.string - assert a["href"].endswith( - "/simple_primary_key?_size=3&_sort={}".format(expected_col) - ) + assert a["href"].endswith(f"/simple_primary_key?_size=3&_sort={expected_col}") assert ["nofollow"] == a["rel"] assert [ [ @@ -730,11 +726,11 @@ def test_table_html_no_primary_key(app_client): '    '.format( i, i ), - ''.format(i), - ''.format(i), - ''.format(i), - ''.format(i), - ''.format(i), + f'', + f'', + f'', + f'', + f'', ] for i in range(1, 51) ] @@ -782,8 +778,8 @@ def test_table_html_compound_primary_key(app_client): for expected_col, th in zip(("pk1", "pk2", "content"), ths[1:]): a = th.find("a") assert expected_col == a.string - assert th["class"] == ["col-{}".format(expected_col)] - assert a["href"].endswith("/compound_primary_key?_sort={}".format(expected_col)) + assert th["class"] == [f"col-{expected_col}"] + assert a["href"].endswith(f"/compound_primary_key?_sort={expected_col}") expected = [ [ '', @@ -1100,9 +1096,7 @@ def test_404(app_client, path): response = app_client.get(path) assert 404 == response.status assert ( - 'Edit SQL' if expected: assert expected_link in response.text else: @@ -1555,10 +1548,10 @@ def test_navigation_menu_links( for link in should_have_links: assert ( details.find("a", {"href": link}) is not None - ), "{} expected but missing from nav menu".format(link) + ), f"{link} expected but missing from nav menu" if should_not_have_links: for link in should_not_have_links: assert ( details.find("a", {"href": link}) is None - ), "{} found but should not have been in nav menu".format(link) + ), f"{link} found but should not have been in nav menu" diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index a56d735b..89290911 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -157,7 +157,7 @@ def test_database_hashed(app_client_with_hash, base_url): ds._config["base_url"] = base_url db_hash = ds.get_database("fixtures").hash assert len(db_hash) == 64 - expected = "{}fixtures-{}".format(base_url, db_hash[:7]) + expected = f"{base_url}fixtures-{db_hash[:7]}" assert ds.urls.database("fixtures") == expected assert ds.urls.table("fixtures", "name") == expected + "/name" assert ds.urls.query("fixtures", "name") == expected + "/name" diff --git a/tests/test_messages.py b/tests/test_messages.py index 830244e1..3af5439a 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -11,7 +11,7 @@ import pytest ], ) def test_add_message_sets_cookie(app_client, qs, expected): - response = app_client.get("/fixtures.message?{}".format(qs)) + response = app_client.get(f"/fixtures.message?{qs}") signed = response.cookies["ds_messages"] decoded = app_client.ds.unsign(signed, "messages") assert expected == decoded diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 3819c872..51faeccb 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -34,7 +34,7 @@ def test_plugin_hooks_have_tests(plugin_hook): for test in tests_in_this_module: if plugin_hook in test: ok = True - assert ok, "Plugin hook is missing tests: {}".format(plugin_hook) + assert ok, f"Plugin hook is missing tests: {plugin_hook}" def test_hook_plugins_dir_plugin_prepare_connection(app_client): @@ -398,7 +398,7 @@ def view_names_client(tmp_path_factory): def test_view_names(view_names_client, path, view_name): response = view_names_client.get(path) assert response.status == 200 - assert "view_name:{}".format(view_name) == response.text + assert f"view_name:{view_name}" == response.text def test_hook_register_output_renderer_no_parameters(app_client): @@ -659,7 +659,7 @@ def test_hook_register_routes_csrftoken(restore_working_directory, tmpdir_factor with make_app_client(template_dir=templates) as client: response = client.get("/csrftoken-form/") expected_token = client.ds._last_request.scope["csrftoken"]() - assert "CSRFTOKEN: {}".format(expected_token) == response.text + assert f"CSRFTOKEN: {expected_token}" == response.text def test_hook_register_routes_asgi(app_client): @@ -793,14 +793,14 @@ def test_hook_table_actions(app_client, table_or_view): return [] return [{"label": a.text, "href": a["href"]} for a in details.select("a")] - response = app_client.get("/fixtures/{}".format(table_or_view)) + response = app_client.get(f"/fixtures/{table_or_view}") assert get_table_actions_links(response.text) == [] - response_2 = app_client.get("/fixtures/{}?_bot=1".format(table_or_view)) + response_2 = app_client.get(f"/fixtures/{table_or_view}?_bot=1") assert get_table_actions_links(response_2.text) == [ {"label": "From async", "href": "/"}, {"label": "Database: fixtures", "href": "/"}, - {"label": "Table: {}".format(table_or_view), "href": "/"}, + {"label": f"Table: {table_or_view}", "href": "/"}, ] diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index a4eca49f..e629bba0 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -62,7 +62,7 @@ Service name: input-service tag = "gcr.io/myproject/datasette" mock_call.assert_has_calls( [ - mock.call("gcloud builds submit --tag {}".format(tag), shell=True), + mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( tag @@ -86,10 +86,10 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] ) assert 0 == result.exit_code - tag = "gcr.io/{}/datasette".format(mock_output.return_value) + tag = f"gcr.io/{mock_output.return_value}/datasette" mock_call.assert_has_calls( [ - mock.call("gcloud builds submit --tag {}".format(tag), shell=True), + mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test".format( tag @@ -129,10 +129,10 @@ def test_publish_cloudrun_memory( assert 2 == result.exit_code return assert 0 == result.exit_code - tag = "gcr.io/{}/datasette".format(mock_output.return_value) + tag = f"gcr.io/{mock_output.return_value}/datasette" mock_call.assert_has_calls( [ - mock.call("gcloud builds submit --tag {}".format(tag), shell=True), + mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --memory {}".format( tag, memory diff --git a/tests/test_utils.py b/tests/test_utils.py index 2d2ff52d..07e6f870 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -457,7 +457,7 @@ def test_check_connection_passes(): def test_call_with_supported_arguments(): def foo(a, b): - return "{}+{}".format(a, b) + return f"{a}+{b}" assert "1+2" == utils.call_with_supported_arguments(foo, a=1, b=2) assert "1+2" == utils.call_with_supported_arguments(foo, a=1, b=2, c=3) diff --git a/update-docs-help.py b/update-docs-help.py index c007e23c..3a192575 100644 --- a/update-docs-help.py +++ b/update-docs-help.py @@ -16,7 +16,7 @@ def update_help_includes(): for name, filename in includes: runner = CliRunner() result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) - actual = "$ datasette {} --help\n\n{}".format(name, result.output) + actual = f"$ datasette {name} --help\n\n{result.output}" actual = actual.replace("Usage: cli ", "Usage: datasette ") open(docs_path / filename, "w").write(actual) From 4bac9f18f9d04e5ed10f072502bcc508e365438e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 21 Nov 2020 15:33:04 -0800 Subject: [PATCH 0168/1866] Fix off-screen action menu bug, refs #1084 --- datasette/static/app.css | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 675285c1..b9378a9e 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -361,13 +361,13 @@ details .nav-menu-inner { } /* Table/database actions menu */ -.actions-menu-links { +.page-header { position: relative; } .actions-menu-links .dropdown-menu { position: absolute; - top: 2rem; - right: 0; + top: calc(100% + 10px); + left: -10px; } /* Components ============================================================== */ From 3159263f05ac4baf968929d59384d9223a539071 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Nov 2020 12:01:47 -0800 Subject: [PATCH 0169/1866] New --setting to replace --config, closes #992 --- datasette/cli.py | 56 +++++++++++++++++++++++++++++++++-- docs/datasette-serve-help.txt | 5 ++-- tests/test_cli.py | 36 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 99075078..9e696aa8 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -2,6 +2,7 @@ import asyncio import uvicorn import click from click import formatting +from click.types import CompositeParamType from click_default_group import DefaultGroup import json import os @@ -29,6 +30,7 @@ from .version import __version__ class Config(click.ParamType): + # This will be removed in Datasette 1.0 in favour of class Setting name = "config" def convert(self, config, param, ctx): @@ -63,6 +65,39 @@ class Config(click.ParamType): self.fail("Invalid option") +class Setting(CompositeParamType): + name = "setting" + arity = 2 + + def convert(self, config, param, ctx): + name, value = config + if name not in DEFAULT_CONFIG: + self.fail( + f"{name} is not a valid option (--help-config to see all)", + param, + ctx, + ) + return + # Type checking + default = DEFAULT_CONFIG[name] + if isinstance(default, bool): + try: + return name, value_as_boolean(value) + except ValueAsBooleanError: + self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) + return + elif isinstance(default, int): + if not value.isdigit(): + self.fail(f'"{name}" should be an integer', param, ctx) + return + return name, int(value) + elif isinstance(default, str): + return name, value + else: + # Should never happen: + self.fail("Invalid option") + + @click.group(cls=DefaultGroup, default="serve", default_if_no_args=True) @click.version_option(version=__version__) def cli(): @@ -330,7 +365,14 @@ def uninstall(packages, yes): @click.option( "--config", type=Config(), - help="Set config option using configname:value docs.datasette.io/en/stable/config.html", + help="Deprecated: set config option using configname:value. Use --setting instead.", + multiple=True, +) +@click.option( + "--setting", + "settings", + type=Setting(), + help="Setting, see docs.datasette.io/en/stable/config.html", multiple=True, ) @click.option( @@ -372,6 +414,7 @@ def serve( static, memory, config, + settings, secret, root, get, @@ -410,6 +453,15 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) + combined_config = {} + if config: + click.echo( + "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", + err=True, + ) + combined_config.update(config) + combined_config.update(settings) + kwargs = dict( immutables=immutable, cache_headers=not reload, @@ -420,7 +472,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - config=dict(config), + config=combined_config, memory=memory, secret=secret, version_note=version_note, diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 5a63d4c4..bdaf0894 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -25,9 +25,10 @@ Options: --plugins-dir DIRECTORY Path to directory containing custom plugins --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make :memory: database available - --config CONFIG Set config option using configname:value - docs.datasette.io/en/stable/config.html + --config CONFIG Deprecated: set config option using configname:value. Use + --setting instead. + --setting SETTING... Setting, see docs.datasette.io/en/stable/config.html --secret TEXT Secret used for signing secure values, such as signed cookies diff --git a/tests/test_cli.py b/tests/test_cli.py index aa39b0ee..99aea053 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ from .fixtures import ( TestClient as _TestClient, EXPECTED_PLUGINS, ) +import asyncio from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ @@ -17,6 +18,13 @@ import textwrap from unittest import mock +@pytest.fixture +def ensure_eventloop(): + # Workaround for "Event loop is closed" error + if asyncio.get_event_loop().is_closed(): + asyncio.set_event_loop(asyncio.new_event_loop()) + + def test_inspect_cli(app_client): runner = CliRunner() result = runner.invoke(cli, ["inspect", "fixtures.db"]) @@ -115,6 +123,7 @@ def test_metadata_yaml(): static=[], memory=False, config=[], + settings=[], secret=None, root=False, version_note=None, @@ -163,3 +172,30 @@ def test_version(): runner = CliRunner() result = runner.invoke(cli, ["--version"]) assert result.output == f"cli, version {__version__}\n" + + +def test_setting(ensure_eventloop): + runner = CliRunner() + result = runner.invoke( + cli, ["--setting", "default_page_size", "5", "--get", "/-/config.json"] + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["default_page_size"] == 5 + + +def test_setting_type_validation(ensure_eventloop): + runner = CliRunner(mix_stderr=False) + result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) + assert result.exit_code == 2 + assert '"default_page_size" should be an integer' in result.stderr + + +def test_config_deprecated(ensure_eventloop): + # The --config option should show a deprecation message + runner = CliRunner(mix_stderr=False) + result = runner.invoke( + cli, ["--config", "allow_download:off", "--get", "/-/config.json"] + ) + assert result.exit_code == 0 + assert not json.loads(result.output)["allow_download"] + assert "will be deprecated in" in result.stderr From 2a3d5b720b96d5ad79ccad655f6575bb71aae302 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Nov 2020 12:19:14 -0800 Subject: [PATCH 0170/1866] Redirect /-/config to /-/settings, closes #1103 --- datasette/app.py | 20 ++++++++++++++++++-- datasette/default_menu_links.py | 4 ++-- docs/introspection.rst | 8 ++++---- tests/test_api.py | 17 +++++++++++++++-- tests/test_cli.py | 4 ++-- tests/test_config_dir.py | 2 +- tests/test_permissions.py | 2 +- 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b2bdb746..36df6032 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -66,6 +66,7 @@ from .utils.asgi import ( Forbidden, NotFound, Request, + Response, asgi_static, asgi_send, asgi_send_html, @@ -884,8 +885,16 @@ class Datasette: r"/-/plugins(?P(\.json)?)$", ) add_route( - JsonDataView.as_view(self, "config.json", lambda: self._config), - r"/-/config(?P(\.json)?)$", + JsonDataView.as_view(self, "settings.json", lambda: self._config), + r"/-/settings(?P(\.json)?)$", + ) + add_route( + permanent_redirect("/-/settings.json"), + r"/-/config.json", + ) + add_route( + permanent_redirect("/-/settings"), + r"/-/config", ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), @@ -1224,6 +1233,13 @@ def wrap_view(view_fn, datasette): return async_view_fn +def permanent_redirect(path): + return wrap_view( + lambda request, send: Response.redirect(path, status=301), + datasette=None, + ) + + _curly_re = re.compile(r"(\{.*?\})") diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py index 0b135410..56f481ef 100644 --- a/datasette/default_menu_links.py +++ b/datasette/default_menu_links.py @@ -22,8 +22,8 @@ def menu_links(datasette, actor): "label": "Metadata", }, { - "href": datasette.urls.path("/-/config"), - "label": "Config", + "href": datasette.urls.path("/-/settings"), + "label": "Settings", }, { "href": datasette.urls.path("/-/permissions"), diff --git a/docs/introspection.rst b/docs/introspection.rst index 698ba95f..a0402b9d 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -89,10 +89,10 @@ Add ``?all=1`` to include details of the default plugins baked into Datasette. .. _JsonDataView_config: -/-/config ---------- +/-/settings +----------- -Shows the :ref:`config` options for this instance of Datasette. `Config example `_: +Shows the :ref:`config` options for this instance of Datasette. `Settings example `_: .. code-block:: json @@ -110,7 +110,7 @@ Shows the :ref:`config` options for this instance of Datasette. `Config example /-/databases ------------ -Shows currently attached databases. `Databases example `_: +Shows currently attached databases. `Databases example `_: .. code-block:: json diff --git a/tests/test_api.py b/tests/test_api.py index 3d48d350..2bab6c30 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1324,8 +1324,8 @@ def test_versions_json(app_client): assert "compile_options" in response.json["sqlite"] -def test_config_json(app_client): - response = app_client.get("/-/config.json") +def test_settings_json(app_client): + response = app_client.get("/-/settings.json") assert { "default_page_size": 50, "default_facet_size": 30, @@ -1350,6 +1350,19 @@ def test_config_json(app_client): } == response.json +@pytest.mark.parametrize( + "path,expected_redirect", + ( + ("/-/config.json", "/-/settings.json"), + ("/-/config", "/-/settings"), + ), +) +def test_config_redirects_to_settings(app_client, path, expected_redirect): + response = app_client.get(path, allow_redirects=False) + assert response.status == 301 + assert response.headers["Location"] == expected_redirect + + def test_page_size_matching_max_returned_rows( app_client_returned_rows_matches_page_size, ): diff --git a/tests/test_cli.py b/tests/test_cli.py index 99aea053..36b9a092 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -177,7 +177,7 @@ def test_version(): def test_setting(ensure_eventloop): runner = CliRunner() result = runner.invoke( - cli, ["--setting", "default_page_size", "5", "--get", "/-/config.json"] + cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] ) assert result.exit_code == 0, result.output assert json.loads(result.output)["default_page_size"] == 5 @@ -194,7 +194,7 @@ def test_config_deprecated(ensure_eventloop): # The --config option should show a deprecation message runner = CliRunner(mix_stderr=False) result = runner.invoke( - cli, ["--config", "allow_download:off", "--get", "/-/config.json"] + cli, ["--config", "allow_download:off", "--get", "/-/settings.json"] ) assert result.exit_code == 0 assert not json.loads(result.output)["allow_download"] diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 15c7a5c4..34bd1d7e 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -86,7 +86,7 @@ def test_metadata(config_dir_client): def test_config(config_dir_client): - response = config_dir_client.get("/-/config.json") + response = config_dir_client.get("/-/settings.json") assert 200 == response.status assert 60 == response.json["default_cache_ttl"] diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 60883eef..3b7e1654 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -378,7 +378,7 @@ def view_instance_client(): "/-/metadata", "/-/versions", "/-/plugins", - "/-/config", + "/-/settings", "/-/threads", "/-/databases", "/-/actor", From 33eadb8782d5b3e179df7dfa08f6d376ded2acd3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Nov 2020 12:37:29 -0800 Subject: [PATCH 0171/1866] config.json is now settings.json, closes #1104 --- datasette/app.py | 7 +++++-- datasette/cli.py | 3 +++ datasette/utils/__init__.py | 4 ++++ docs/config.rst | 8 ++++---- tests/test_config_dir.py | 17 ++++++++++++++--- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 36df6032..0e42b7c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -45,6 +45,7 @@ from .database import Database, QueryInterrupted from .utils import ( PrefixedUrlString, + StartupError, async_call_with_supported_arguments, await_me_maybe, call_with_supported_arguments, @@ -265,8 +266,10 @@ class Datasette: if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str((config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] - if config_dir and (config_dir / "config.json").exists() and not config: - config = json.load((config_dir / "config.json").open()) + if config_dir and (config_dir / "config.json").exists(): + raise StartupError("config.json should be renamed to settings.json") + if config_dir and (config_dir / "settings.json").exists() and not config: + config = json.load((config_dir / "settings.json").open()) self._config = dict(DEFAULT_CONFIG, **(config or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/datasette/cli.py b/datasette/cli.py index 9e696aa8..95e1418c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -14,6 +14,7 @@ from runpy import run_module import webbrowser from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm from .utils import ( + StartupError, check_connection, parse_metadata, ConnectionProblem, @@ -488,6 +489,8 @@ def serve( ds = Datasette(files, **kwargs) except SpatialiteNotFound: raise click.ClickException("Could not find SpatiaLite extension") + except StartupError as e: + raise click.ClickException(e.args[0]) if return_instance: # Private utility mechanism for writing unit tests diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 02b59b2b..d62302e9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1027,3 +1027,7 @@ class PrefixedUrlString(str): return method.__get__(self) else: return super().__getattribute__(name) + + +class StartupError(Exception): + pass diff --git a/docs/config.rst b/docs/config.rst index 0883e532..27b73d44 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -50,15 +50,15 @@ The files that can be included in this directory are as follows. All are optiona * ``*.db`` - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect`` - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running -* ``config.json`` - settings that would normally be passed using ``--config`` - here they should be stored as a JSON object of key/value pairs +* ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs * ``templates/`` - a directory containing :ref:`customization_custom_templates` * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off` * ``static/`` - a directory containing static files - these will be served from ``/static/filename.txt``, see :ref:`customization_static_files` -Configuration options ---------------------- +Settings +-------- -The followig options can be set using ``--config name:value``, or by storing them in the ``config.json`` file for use with :ref:`config_dir`. +The following options can be set using ``--setting name value``, or by storing them in the ``settings.json`` file for use with :ref:`config_dir`. default_page_size ~~~~~~~~~~~~~~~~~ diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 34bd1d7e..cd158474 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -3,7 +3,9 @@ import pytest import sqlite3 from datasette.app import Datasette +from datasette.cli import cli from .fixtures import TestClient as _TestClient +from click.testing import CliRunner PLUGIN = """ from datasette import hookimpl @@ -15,7 +17,7 @@ def extra_template_vars(): } """ METADATA = {"title": "This is from metadata"} -CONFIG = { +SETTINGS = { "default_cache_ttl": 60, } CSS = """ @@ -44,7 +46,7 @@ def config_dir_client(tmp_path_factory): (static_dir / "hello.css").write_text(CSS, "utf-8") (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") - (config_dir / "config.json").write_text(json.dumps(CONFIG), "utf-8") + (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") for dbname in ("demo.db", "immutable.db"): db = sqlite3.connect(str(config_dir / dbname)) @@ -85,12 +87,21 @@ def test_metadata(config_dir_client): assert METADATA == response.json -def test_config(config_dir_client): +def test_settings(config_dir_client): response = config_dir_client.get("/-/settings.json") assert 200 == response.status assert 60 == response.json["default_cache_ttl"] +def test_error_on_config_json(tmp_path_factory): + config_dir = tmp_path_factory.mktemp("config-dir") + (config_dir / "config.json").write_text(json.dumps(SETTINGS), "utf-8") + runner = CliRunner(mix_stderr=False) + result = runner.invoke(cli, [str(config_dir), "--get", "/-/settings.json"]) + assert result.exit_code == 1 + assert "config.json should be renamed to settings.json" in result.stderr + + def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status From 5a77f7a6494c74372bedfef6185e1beed7bea5dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Nov 2020 13:22:33 -0800 Subject: [PATCH 0172/1866] Updated docs renaming config to settings - config.html is now settings.html - ConfigOption in app.py is now Setting - updated documentation unit tests Refs #1106 --- .github/workflows/deploy-latest.yml | 4 +- datasette/app.py | 56 +++++++-------- datasette/cli.py | 12 ++-- docs/changelog.rst | 18 ++--- docs/csv_export.rst | 6 +- docs/deploying.rst | 8 +-- docs/index.rst | 2 +- docs/internals.rst | 6 +- docs/introspection.rst | 2 +- docs/pages.rst | 2 +- docs/performance.rst | 6 +- docs/plugin_hooks.rst | 2 +- docs/publish.rst | 4 +- docs/{config.rst => settings.rst} | 108 ++++++++++++++++------------ docs/spatialite.rst | 2 +- docs/writing_plugins.rst | 2 +- tests/test_docs.py | 10 +-- 17 files changed, 131 insertions(+), 119 deletions(-) rename docs/{config.rst => settings.rst} (80%) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 73b97a19..7a41bda2 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -53,11 +53,11 @@ jobs: --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--config template_debug:1" \ + --extra-options="--setting template_debug 1" \ --service=datasette-latest # Deploy docs.db to a different service datasette publish cloudrun docs.db \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--config template_debug:1" \ + --extra-options="--setting template_debug 1" \ --service=datasette-docs-latest diff --git a/datasette/app.py b/datasette/app.py index 0e42b7c6..3bb6ce79 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -82,91 +82,85 @@ app_root = Path(__file__).parent.parent MEMORY = object() -ConfigOption = collections.namedtuple("ConfigOption", ("name", "default", "help")) -CONFIG_OPTIONS = ( - ConfigOption("default_page_size", 100, "Default page size for the table view"), - ConfigOption( +Setting = collections.namedtuple("Setting", ("name", "default", "help")) +SETTINGS = ( + Setting("default_page_size", 100, "Default page size for the table view"), + Setting( "max_returned_rows", 1000, "Maximum rows that can be returned from a table or custom query", ), - ConfigOption( + Setting( "num_sql_threads", 3, "Number of threads in the thread pool for executing SQLite queries", ), - ConfigOption( - "sql_time_limit_ms", 1000, "Time limit for a SQL query in milliseconds" - ), - ConfigOption( + Setting("sql_time_limit_ms", 1000, "Time limit for a SQL query in milliseconds"), + Setting( "default_facet_size", 30, "Number of values to return for requested facets" ), - ConfigOption( - "facet_time_limit_ms", 200, "Time limit for calculating a requested facet" - ), - ConfigOption( + Setting("facet_time_limit_ms", 200, "Time limit for calculating a requested facet"), + Setting( "facet_suggest_time_limit_ms", 50, "Time limit for calculating a suggested facet", ), - ConfigOption( + Setting( "hash_urls", False, "Include DB file contents hash in URLs, for far-future caching", ), - ConfigOption( + Setting( "allow_facet", True, "Allow users to specify columns to facet using ?_facet= parameter", ), - ConfigOption( + Setting( "allow_download", True, "Allow users to download the original SQLite database files", ), - ConfigOption("suggest_facets", True, "Calculate and display suggested facets"), - ConfigOption( + Setting("suggest_facets", True, "Calculate and display suggested facets"), + Setting( "default_cache_ttl", 5, "Default HTTP cache TTL (used in Cache-Control: max-age= header)", ), - ConfigOption( + Setting( "default_cache_ttl_hashed", 365 * 24 * 60 * 60, "Default HTTP cache TTL for hashed URL pages", ), - ConfigOption( - "cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)" - ), - ConfigOption( + Setting("cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)"), + Setting( "allow_csv_stream", True, "Allow .csv?_stream=1 to download all rows (ignoring max_returned_rows)", ), - ConfigOption( + Setting( "max_csv_mb", 100, "Maximum size allowed for CSV export in MB - set 0 to disable this limit", ), - ConfigOption( + Setting( "truncate_cells_html", 2048, "Truncate cells longer than this in HTML table view - set 0 to disable", ), - ConfigOption( + Setting( "force_https_urls", False, "Force URLs in API output to always use https:// protocol", ), - ConfigOption( + Setting( "template_debug", False, "Allow display of template debug information with ?_context=1", ), - ConfigOption("base_url", "/", "Datasette URLs should use this base path"), + Setting("base_url", "/", "Datasette URLs should use this base path"), ) -DEFAULT_CONFIG = {option.name: option.default for option in CONFIG_OPTIONS} +DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} async def favicon(request, send): @@ -270,7 +264,7 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: config = json.load((config_dir / "settings.json").open()) - self._config = dict(DEFAULT_CONFIG, **(config or {})) + self._config = dict(DEFAULT_SETTINGS, **(config or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( @@ -358,7 +352,7 @@ class Datasette: def config_dict(self): # Returns a fully resolved config dictionary, useful for templates - return {option.name: self.config(option.name) for option in CONFIG_OPTIONS} + return {option.name: self.config(option.name) for option in SETTINGS} def metadata(self, key=None, database=None, table=None, fallback=True): """ diff --git a/datasette/cli.py b/datasette/cli.py index 95e1418c..5feab51e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -12,7 +12,7 @@ from subprocess import call import sys from runpy import run_module import webbrowser -from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm +from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, pm from .utils import ( StartupError, check_connection, @@ -39,7 +39,7 @@ class Config(click.ParamType): self.fail(f'"{config}" should be name:value', param, ctx) return name, value = config.split(":", 1) - if name not in DEFAULT_CONFIG: + if name not in DEFAULT_SETTINGS: self.fail( f"{name} is not a valid option (--help-config to see all)", param, @@ -47,7 +47,7 @@ class Config(click.ParamType): ) return # Type checking - default = DEFAULT_CONFIG[name] + default = DEFAULT_SETTINGS[name] if isinstance(default, bool): try: return name, value_as_boolean(value) @@ -72,7 +72,7 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config - if name not in DEFAULT_CONFIG: + if name not in DEFAULT_SETTINGS: self.fail( f"{name} is not a valid option (--help-config to see all)", param, @@ -80,7 +80,7 @@ class Setting(CompositeParamType): ) return # Type checking - default = DEFAULT_CONFIG[name] + default = DEFAULT_SETTINGS[name] if isinstance(default, bool): try: return name, value_as_boolean(value) @@ -432,7 +432,7 @@ def serve( formatter.write_dl( [ (option.name, f"{option.help} (default={option.default})") - for option in CONFIG_OPTIONS + for option in SETTINGS ] ) click.echo(formatter.getvalue()) diff --git a/docs/changelog.rst b/docs/changelog.rst index 34bd95d4..2916b373 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -49,7 +49,7 @@ The new :ref:`internals_datasette_urls` family of methods can be used to generat Running Datasette behind a proxy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`config_base_url` configuration option is designed to help run Datasette on a specific path behind a proxy - for example if you want to run an instance of Datasette at ``/my-datasette/`` within your existing site's URL hierarchy, proxied behind nginx or Apache. +The :ref:`setting_base_url` configuration option is designed to help run Datasette on a specific path behind a proxy - for example if you want to run an instance of Datasette at ``/my-datasette/`` within your existing site's URL hierarchy, proxied behind nginx or Apache. Support for this configuration option has been greatly improved (`#1023 `__), and guidelines for using it are now available in a new documentation section on :ref:`deploying_proxy`. (`#1027 `__) @@ -353,9 +353,9 @@ Signed values and secrets Both flash messages and user authentication needed a way to sign values and set signed cookies. Two new methods are now available for plugins to take advantage of this mechanism: :ref:`datasette_sign` and :ref:`datasette_unsign`. -Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`config_secret` for more details. +Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`settings_secret` for more details. -You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`config_publish_secrets`. +You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`settings_publish_secrets`. Plugins can now sign values and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods. @@ -450,7 +450,7 @@ A small release which provides improved internal methods for use in plugins, alo You can now create :ref:`custom pages ` within your Datasette instance using a custom template file. For example, adding a template file called ``templates/pages/about.html`` will result in a new page being served at ``/about`` on your instance. See the :ref:`custom pages documentation ` for full details, including how to return custom HTTP headers, redirects and status codes. (`#648 `__) -:ref:`config_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: +:ref:`settings_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: $ datasette one.db two.db \ --metadata.json \ @@ -480,7 +480,7 @@ Also in this release: * Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. See :ref:`metadata_yaml`. (`#713 `__) * Removed support for ``datasette publish now``, which used the the now-retired Zeit Now v1 hosting platform. A new plugin, `datasette-publish-now `__, can be installed to publish data to Zeit (`now Vercel `__) Now v2. (`#710 `__) * Fixed a bug where the ``extra_template_vars(request, view_name)`` plugin hook was not receiving the correct ``view_name``. (`#716 `__) -* Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`config_template_debug`). (`#693 `__) +* Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`settings_template_debug`). (`#693 `__) * Fixed a bug where the "templates considered" HTML comment was no longer being displayed. (`#689 `__) * Fixed a ``datasette publish`` bug where ``--plugin-secret`` would over-ride plugin configuration in the provided ``metadata.json`` file. (`#724 `__) * Added a new CSS class for customizing the canned query page. (`#727 `__) @@ -490,7 +490,7 @@ Also in this release: 0.39 (2020-03-24) ----------------- -* New :ref:`config_base_url` configuration setting for serving up the correct links while running Datasette under a different URL prefix. (`#394 `__) +* New :ref:`setting_base_url` configuration setting for serving up the correct links while running Datasette under a different URL prefix. (`#394 `__) * New metadata settings ``"sort"`` and ``"sort_desc"`` for setting the default sort order for a table. See :ref:`metadata_default_sort`. (`#702 `__) * Sort direction arrow now displays by default on the primary key. This means you only have to click once (not twice) to sort in reverse order. (`#677 `__) * New ``await Request(scope, receive).post_vars()`` method for accessing POST form variables. (`#700 `__) @@ -565,7 +565,7 @@ Also in this release: * asyncio task information is now included on the ``/-/threads`` debug page * Bumped Uvicorn dependency 0.11 * You can now use ``--port 0`` to listen on an available port -* New :ref:`config_template_debug` setting for debugging templates, e.g. https://latest.datasette.io/fixtures/roadside_attractions?_context=1 (`#654 `__) +* New :ref:`settings_template_debug` setting for debugging templates, e.g. https://latest.datasette.io/fixtures/roadside_attractions?_context=1 (`#654 `__) .. _v0_32: @@ -1000,7 +1000,7 @@ Check out the :ref:`CSV export documentation ` for more details, or try the feature out on https://fivethirtyeight.datasettes.com/fivethirtyeight/bechdel%2Fmovies -If your table has more than :ref:`config_max_returned_rows` (default 1,000) +If your table has more than :ref:`settings_max_returned_rows` (default 1,000) Datasette provides the option to *stream all rows*. This option takes advantage of async Python and Datasette's efficient :ref:`pagination ` to iterate through the entire matching result set and stream it back as a @@ -1020,7 +1020,7 @@ table, using the new ``_labels=on`` querystring option. See New configuration settings ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Datasette's :ref:`config` now also supports boolean settings. A number of new +Datasette's :ref:`settings` now also supports boolean settings. A number of new configuration options have been added: * ``num_sql_threads`` - the number of threads used to execute SQLite queries. Defaults to 3. diff --git a/docs/csv_export.rst b/docs/csv_export.rst index b5cc599a..704cc19d 100644 --- a/docs/csv_export.rst +++ b/docs/csv_export.rst @@ -23,7 +23,7 @@ file, which looks like this and has the following options: the ``city_id`` column is accompanied by a ``city_id_label`` column. * **stream all rows** - by default CSV files only contain the first - :ref:`config_max_returned_rows` records. This option will cause Datasette to + :ref:`settings_max_returned_rows` records. This option will cause Datasette to loop through every matching record and return them as a single CSV file. You can try that out on https://latest.datasette.io/fixtures/facetable?_size=4 @@ -40,9 +40,9 @@ Since databases can get pretty large, by default this option is capped at 100MB if a table returns more than 100MB of data the last line of the CSV will be a truncation error message. -You can increase or remove this limit using the :ref:`config_max_csv_mb` config +You can increase or remove this limit using the :ref:`settings_max_csv_mb` config setting. You can also disable the CSV export feature entirely using -:ref:`config_allow_csv_stream`. +:ref:`settings_allow_csv_stream`. A note on URLs -------------- diff --git a/docs/deploying.rst b/docs/deploying.rst index 3eeaaad8..4ca0e82a 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -58,7 +58,7 @@ Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Data $ python3 -c 'import secrets; print(secrets.token_hex(32))' -This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. +This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`settings_dir` for details. You can start the Datasette process running using the following:: @@ -101,7 +101,7 @@ The ``Procfile`` lets the hosting platform know how to run the command that serv web: datasette . -h 0.0.0.0 -p $PORT --cors -The ``$PORT`` environment variable is provided by the hosting platform. ``--cors`` enables CORS requests from JavaScript running on other websites to your domain - omit this if you don't want to allow CORS. You can add additional Datasette :ref:`config` options here too. +The ``$PORT`` environment variable is provided by the hosting platform. ``--cors`` enables CORS requests from JavaScript running on other websites to your domain - omit this if you don't want to allow CORS. You can add additional Datasette :ref:`settings` options here too. These two files should be enough to deploy Datasette on any host that supports buildpacks. Datasette will serve any SQLite files that are included in the root directory of the application. @@ -118,9 +118,9 @@ Running Datasette behind a proxy You may wish to run Datasette behind an Apache or nginx proxy, using a path within your existing site. -You can use the :ref:`config_base_url` configuration setting to tell Datasette to serve traffic with a specific URL prefix. For example, you could run Datasette like this:: +You can use the :ref:`setting_base_url` configuration setting to tell Datasette to serve traffic with a specific URL prefix. For example, you could run Datasette like this:: - datasette my-database.db --config base_url:/my-datasette/ -p 8009 + datasette my-database.db --setting base_url /my-datasette/ -p 8009 This will run Datasette with the following URLs: diff --git a/docs/index.rst b/docs/index.rst index 6b55da8c..ff8db04b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,7 +51,7 @@ Contents full_text_search spatialite metadata - config + settings introspection custom_templates plugins diff --git a/docs/internals.rst b/docs/internals.rst index 92496490..cec1268f 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -350,7 +350,7 @@ Returns the absolute URL for the given path, including the protocol and host. Fo absolute_url = datasette.absolute_url(request, "/dbname/table.json") # Would return "http://localhost:8001/dbname/table.json" -The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`config_force_https_urls` configuration setting is taken into account. +The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`settings_force_https_urls` configuration setting is taken into account. .. _internals_datasette_client: @@ -397,7 +397,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example: ) ).json() -``datasette.client`` methods automatically take the current :ref:`config_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path. +``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path. For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. @@ -406,7 +406,7 @@ For documentation on available ``**kwargs`` options and the shape of the HTTPX R datasette.urls -------------- -The ``datasette.urls`` object contains methods for building URLs to pages within Datasette. Plugins should use this to link to pages, since these methods take into account any :ref:`config_base_url` configuration setting that might be in effect. +The ``datasette.urls`` object contains methods for building URLs to pages within Datasette. Plugins should use this to link to pages, since these methods take into account any :ref:`setting_base_url` configuration setting that might be in effect. ``datasette.urls.instance(format=None)`` Returns the URL to the Datasette instance root page. This is usually ``"/"``. diff --git a/docs/introspection.rst b/docs/introspection.rst index a0402b9d..d1a0a854 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -92,7 +92,7 @@ Add ``?all=1`` to include details of the default plugins baked into Datasette. /-/settings ----------- -Shows the :ref:`config` options for this instance of Datasette. `Settings example `_: +Shows the :ref:`settings` for this instance of Datasette. `Settings example `_: .. code-block:: json diff --git a/docs/pages.rst b/docs/pages.rst index db970ead..5f77bec7 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -66,7 +66,7 @@ Row Every row in every Datasette table has its own URL. This means individual records can be linked to directly. -Table cells with extremely long text contents are truncated on the table view according to the :ref:`config_truncate_cells_html` setting. If a cell has been truncated the full length version of that cell will be available on the row page. +Table cells with extremely long text contents are truncated on the table view according to the :ref:`settings_truncate_cells_html` setting. If a cell has been truncated the full length version of that cell will be available on the row page. Rows which are the targets of foreign key references from other tables will show a link to a filtered search for all records that reference that row. Here's an example from the Registers of Members Interests database: diff --git a/docs/performance.rst b/docs/performance.rst index d7f852d5..1d24adce 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -56,7 +56,7 @@ Using a caching proxy in this way could enable a Datasette-backed visualization Datasette's integration with HTTP caches can be enabled using a combination of configuration options and querystring arguments. -The :ref:`config_default_cache_ttl` setting sets the default HTTP cache TTL for all Datasette pages. This is 5 seconds unless you change it - you can set it to 0 if you wish to disable HTTP caching entirely. +The :ref:`settings_default_cache_ttl` setting sets the default HTTP cache TTL for all Datasette pages. This is 5 seconds unless you change it - you can set it to 0 if you wish to disable HTTP caching entirely. You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` querystring parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query. @@ -65,9 +65,9 @@ Hashed URL mode When you open a database file in immutable mode using the ``-i`` option, Datasette calculates a SHA-256 hash of the contents of that file on startup. This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future. This results in URLs that can then be cached indefinitely by both browsers and caching proxies - an enormous potential performance optimization. -You can enable these hashed URLs in two ways: using the :ref:`config_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` querystring parameter (which only applies to the current request). +You can enable these hashed URLs in two ways: using the :ref:`settings_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` querystring parameter (which only applies to the current request). -With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`config_default_cache_ttl_hashed` which defaults to 365 days. +With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`settings_default_cache_ttl_hashed` which defaults to 365 days. Since these responses are cached for a long time, you may wish to build API clients against the non-hashed version of these URLs. These 302 redirects are served extremely quickly, so this should still be a performant way to work against the Datasette API. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 8407a259..72b09367 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1020,7 +1020,7 @@ This example adds a new menu item but only if the signed in user is ``"root"``: {"href": datasette.urls.path("/-/edit-schema"), "label": "Edit schema"}, ] -Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`config_base_url` setting into account. +Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. .. _plugin_hook_table_actions: diff --git a/docs/publish.rst b/docs/publish.rst index a905ac92..d5015e21 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -135,7 +135,7 @@ If you have docker installed (e.g. using `Docker for Mac 79e1dc9af1c1 @@ -154,7 +154,7 @@ Here's example output for the package command:: Step 6/7 : EXPOSE 8001 ---> Using cache ---> 8e83844b0fed - Step 7/7 : CMD datasette serve parlgov.db --port 8001 --inspect-file inspect-data.json --config sql_time_limit_ms:2500 + Step 7/7 : CMD datasette serve parlgov.db --port 8001 --inspect-file inspect-data.json --setting sql_time_limit_ms 2500 ---> Using cache ---> 1bd380ea8af3 Successfully built 1bd380ea8af3 diff --git a/docs/config.rst b/docs/settings.rst similarity index 80% rename from docs/config.rst rename to docs/settings.rst index 27b73d44..350fd048 100644 --- a/docs/config.rst +++ b/docs/settings.rst @@ -1,20 +1,19 @@ -.. _config: +.. _settings: -Configuration -============= +Settings +======== -Using \-\-config ----------------- +Using \-\-setting +----------------- -Datasette provides a number of configuration options. These can be set using the ``--config name:value`` option to ``datasette serve``. +Datasette supports a number of settings. These can be set using the ``--setting name value`` option to ``datasette serve``. -You can set multiple configuration options at once like this:: +You can set multiple settings at once like this:: datasette mydatabase.db \ - --config default_page_size:50 \ - --config sql_time_limit_ms:3500 \ - --config max_returned_rows:2000 - + --setting default_page_size 50 \ + --setting sql_time_limit_ms 3500 \ + --setting max_returned_rows 2000 .. _config_dir: @@ -60,12 +59,16 @@ Settings The following options can be set using ``--setting name value``, or by storing them in the ``settings.json`` file for use with :ref:`config_dir`. +.. _setting_default_page_size: + default_page_size ~~~~~~~~~~~~~~~~~ -The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` querystring parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--config`` like so:: +The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` querystring parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--setting`` like so:: - datasette mydatabase.db --config default_page_size:50 + datasette mydatabase.db --setting default_page_size 50 + +.. _setting_sql_time_limit_ms: sql_time_limit_ms ~~~~~~~~~~~~~~~~~ @@ -74,7 +77,7 @@ By default, queries have a time limit of one second. If a query takes longer tha If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds:: - datasette mydatabase.db --config sql_time_limit_ms:3500 + datasette mydatabase.db --setting sql_time_limit_ms 3500 You can optionally set a lower time limit for an individual query using the ``?_timelimit=100`` querystring argument:: @@ -82,7 +85,7 @@ You can optionally set a lower time limit for an individual query using the ``?_ This would set the time limit to 100ms for that specific query. This feature is useful if you are working with databases of unknown size and complexity - a query that might make perfect sense for a smaller table could take too long to execute on a table with millions of rows. By setting custom time limits you can execute queries "optimistically" - e.g. give me an exact count of rows matching this query but only if it takes less than 100ms to calculate. -.. _config_max_returned_rows: +.. _setting_max_returned_rows: max_returned_rows ~~~~~~~~~~~~~~~~~ @@ -91,7 +94,9 @@ Datasette returns a maximum of 1,000 rows of data at a time. If you execute a qu You can increase or decrease this limit like so:: - datasette mydatabase.db --config max_returned_rows:2000 + datasette mydatabase.db --setting max_returned_rows 2000 + +.. _setting_num_sql_threads: num_sql_threads ~~~~~~~~~~~~~~~ @@ -100,7 +105,9 @@ Maximum number of threads in the thread pool Datasette uses to execute SQLite qu :: - datasette mydatabase.db --config num_sql_threads:10 + datasette mydatabase.db --setting num_sql_threads 10 + +.. _setting_allow_facet: allow_facet ~~~~~~~~~~~ @@ -111,21 +118,27 @@ This is enabled by default. If disabled, facets will still be displayed if they Here's how to disable this feature:: - datasette mydatabase.db --config allow_facet:off + datasette mydatabase.db --setting allow_facet off + +.. _setting_default_facet_size: default_facet_size ~~~~~~~~~~~~~~~~~~ The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this:: - datasette mydatabase.db --config default_facet_size:50 + datasette mydatabase.db --setting default_facet_size 50 + +.. _setting_facet_time_limit_ms: facet_time_limit_ms ~~~~~~~~~~~~~~~~~~~ This is the time limit Datasette allows for calculating a facet, which defaults to 200ms:: - datasette mydatabase.db --config facet_time_limit_ms:1000 + datasette mydatabase.db --setting facet_time_limit_ms 1000 + +.. _setting_facet_suggest_time_limit_ms: facet_suggest_time_limit_ms ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -134,23 +147,27 @@ When Datasette calculates suggested facets it needs to run a SQL query for every You can increase this time limit like so:: - datasette mydatabase.db --config facet_suggest_time_limit_ms:500 + datasette mydatabase.db --setting facet_suggest_time_limit_ms 500 + +.. _setting_suggest_facets: suggest_facets ~~~~~~~~~~~~~~ Should Datasette calculate suggested facets? On by default, turn this off like so:: - datasette mydatabase.db --config suggest_facets:off + datasette mydatabase.db --setting suggest_facets off + +.. _setting_allow_download: allow_download ~~~~~~~~~~~~~~ Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default - to disable database downloads, use the following:: - datasette mydatabase.db --config allow_download:off + datasette mydatabase.db --setting allow_download off -.. _config_default_cache_ttl: +.. _setting_default_cache_ttl: default_cache_ttl ~~~~~~~~~~~~~~~~~ @@ -159,19 +176,20 @@ Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-ag :: - datasette mydatabase.db --config default_cache_ttl:60 + datasette mydatabase.db --setting default_cache_ttl 60 -.. _config_default_cache_ttl_hashed: +.. _setting_default_cache_ttl_hashed: default_cache_ttl_hashed ~~~~~~~~~~~~~~~~~~~~~~~~ -Default HTTP caching max-age for responses served using using the :ref:`hashed-urls mechanism `. Defaults to 365 days (31536000 seconds). +Default HTTP caching max-age for responses served using using the :ref:`hashed-urls mechanism `. Defaults to 365 days (31536000 seconds). :: - datasette mydatabase.db --config default_cache_ttl_hashed:10000 + datasette mydatabase.db --setting default_cache_ttl_hashed 10000 +.. _setting_cache_size_kb: cache_size_kb ~~~~~~~~~~~~~ @@ -180,9 +198,9 @@ Sets the amount of memory SQLite uses for its `per-connection cache Date: Tue, 24 Nov 2020 14:06:32 -0800 Subject: [PATCH 0173/1866] Renamed datasette.config() to .setting(), closes #1107 --- datasette/app.py | 32 ++++++++++++------------ datasette/facets.py | 22 ++++++++-------- datasette/url_builder.py | 4 +-- datasette/views/base.py | 10 ++++---- datasette/views/database.py | 4 +-- datasette/views/table.py | 10 ++++---- docs/internals.rst | 16 +++++++++++- tests/test_internals_datasette.py | 12 +++++++++ tests/test_internals_datasette_client.py | 12 ++++----- tests/test_internals_urls.py | 24 +++++++++--------- 10 files changed, 86 insertions(+), 60 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3bb6ce79..88d5ecc6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -264,15 +264,15 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: config = json.load((config_dir / "settings.json").open()) - self._config = dict(DEFAULT_SETTINGS, **(config or {})) + self._settings = dict(DEFAULT_SETTINGS, **(config or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( - max_workers=self.config("num_sql_threads") + max_workers=self.setting("num_sql_threads") ) - self.max_returned_rows = self.config("max_returned_rows") - self.sql_time_limit_ms = self.config("sql_time_limit_ms") - self.page_size = self.config("default_page_size") + self.max_returned_rows = self.setting("max_returned_rows") + self.sql_time_limit_ms = self.setting("sql_time_limit_ms") + self.page_size = self.setting("default_page_size") # Execute plugins in constructor, to ensure they are available # when the rest of `datasette inspect` executes if self.plugins_dir: @@ -347,12 +347,12 @@ class Datasette: def remove_database(self, name): self.databases.pop(name) - def config(self, key): - return self._config.get(key, None) + def setting(self, key): + return self._settings.get(key, None) def config_dict(self): # Returns a fully resolved config dictionary, useful for templates - return {option.name: self.config(option.name) for option in SETTINGS} + return {option.name: self.setting(option.name) for option in SETTINGS} def metadata(self, key=None, database=None, table=None, fallback=True): """ @@ -454,8 +454,8 @@ class Datasette: conn.enable_load_extension(True) for extension in self.sqlite_extensions: conn.execute(f"SELECT load_extension('{extension}')") - if self.config("cache_size_kb"): - conn.execute(f"PRAGMA cache_size=-{self.config('cache_size_kb')}") + if self.setting("cache_size_kb"): + conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member pm.hook.prepare_connection(conn=conn, database=database, datasette=self) @@ -567,7 +567,7 @@ class Datasette: def absolute_url(self, request, path): url = urllib.parse.urljoin(request.url, path) - if url.startswith("http://") and self.config("force_https_urls"): + if url.startswith("http://") and self.setting("force_https_urls"): url = "https://" + url[len("http://") :] return url @@ -781,12 +781,12 @@ class Datasette: "extra_js_urls": await self._asset_urls( "extra_js_urls", template, context, request, view_name ), - "base_url": self.config("base_url"), + "base_url": self.setting("base_url"), "csrftoken": request.scope["csrftoken"] if request else lambda: "", }, **extra_template_vars, } - if request and request.args.get("_context") and self.config("template_debug"): + if request and request.args.get("_context") and self.setting("template_debug"): return "
    {}
    ".format( jinja2.escape(json.dumps(template_context, default=repr, indent=4)) ) @@ -882,7 +882,7 @@ class Datasette: r"/-/plugins(?P(\.json)?)$", ) add_route( - JsonDataView.as_view(self, "settings.json", lambda: self._config), + JsonDataView.as_view(self, "settings.json", lambda: self._settings), r"/-/settings(?P(\.json)?)$", ) add_route( @@ -1001,7 +1001,7 @@ class DatasetteRouter: async def route_path(self, scope, receive, send, path): # Strip off base_url if present before routing - base_url = self.ds.config("base_url") + base_url = self.ds.setting("base_url") if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] request = Request(scope, receive) @@ -1016,7 +1016,7 @@ class DatasetteRouter: scope_modifications = {} # Apply force_https_urls, if set if ( - self.ds.config("force_https_urls") + self.ds.setting("force_https_urls") and scope["type"] == "http" and scope.get("scheme") != "https" ): diff --git a/datasette/facets.py b/datasette/facets.py index a818a9e9..8ad5a423 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -136,7 +136,7 @@ class ColumnFacet(Facet): async def suggest(self): row_count = await self.get_row_count() columns = await self.get_columns(self.sql, self.params) - facet_size = self.ds.config("default_facet_size") + facet_size = self.ds.setting("default_facet_size") suggested_facets = [] already_enabled = [c["config"]["simple"] for c in self.get_configs()] for column in columns: @@ -158,7 +158,7 @@ class ColumnFacet(Facet): suggested_facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"), ) num_distinct_values = len(distinct_values) if ( @@ -188,7 +188,7 @@ class ColumnFacet(Facet): qs_pairs = self.get_querystring_pairs() - facet_size = self.ds.config("default_facet_size") + facet_size = self.ds.setting("default_facet_size") for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] @@ -208,7 +208,7 @@ class ColumnFacet(Facet): facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] facet_results[column] = { @@ -290,7 +290,7 @@ class ArrayFacet(Facet): suggested_facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"), log_sql_errors=False, ) types = tuple(r[0] for r in results.rows) @@ -305,7 +305,7 @@ class ArrayFacet(Facet): ), self.params, truncate=False, - custom_time_limit=self.ds.config( + custom_time_limit=self.ds.setting( "facet_suggest_time_limit_ms" ), log_sql_errors=False, @@ -335,7 +335,7 @@ class ArrayFacet(Facet): facet_results = {} facets_timed_out = [] - facet_size = self.ds.config("default_facet_size") + facet_size = self.ds.setting("default_facet_size") for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] @@ -354,7 +354,7 @@ class ArrayFacet(Facet): facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] facet_results[column] = { @@ -421,7 +421,7 @@ class DateFacet(Facet): suggested_facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"), log_sql_errors=False, ) values = tuple(r[0] for r in results.rows) @@ -446,7 +446,7 @@ class DateFacet(Facet): facet_results = {} facets_timed_out = [] args = dict(self.get_querystring_pairs()) - facet_size = self.ds.config("default_facet_size") + facet_size = self.ds.setting("default_facet_size") for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] @@ -467,7 +467,7 @@ class DateFacet(Facet): facet_sql, self.params, truncate=False, - custom_time_limit=self.ds.config("facet_time_limit_ms"), + custom_time_limit=self.ds.setting("facet_time_limit_ms"), ) facet_results_values = [] facet_results[column] = { diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 697f60ae..3034b664 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -10,7 +10,7 @@ class Urls: if not isinstance(path, PrefixedUrlString): if path.startswith("/"): path = path[1:] - path = self.ds.config("base_url") + path + path = self.ds.setting("base_url") + path if format is not None: path = path_with_format(path=path, format=format) return PrefixedUrlString(path) @@ -29,7 +29,7 @@ class Urls: def database(self, database, format=None): db = self.ds.databases[database] - if self.ds.config("hash_urls") and db.hash: + if self.ds.setting("hash_urls") and db.hash: path = self.path(f"{database}-{db.hash[:HASH_LENGTH]}", format=format) else: path = self.path(database, format=format) diff --git a/datasette/views/base.py b/datasette/views/base.py index b3a54bcc..bde8449f 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -230,7 +230,7 @@ class DataView(BaseView): should_redirect += kwargs["as_db"] if ( - (self.ds.config("hash_urls") or "_hash" in request.args) + (self.ds.setting("hash_urls") or "_hash" in request.args) and # Redirect only if database is immutable not self.ds.databases[name].is_mutable @@ -260,7 +260,7 @@ class DataView(BaseView): stream = request.args.get("_stream") if stream: # Some quick sanity checks - if not self.ds.config("allow_csv_stream"): + if not self.ds.setting("allow_csv_stream"): raise BadRequest("CSV streaming is disabled") if request.args.get("_next"): raise BadRequest("_next not allowed for CSV streaming") @@ -296,7 +296,7 @@ class DataView(BaseView): async def stream_fn(r): nonlocal data - writer = csv.writer(LimitedWriter(r, self.ds.config("max_csv_mb"))) + writer = csv.writer(LimitedWriter(r, self.ds.setting("max_csv_mb"))) first = True next = None while first or (next and stream): @@ -566,9 +566,9 @@ class DataView(BaseView): ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): if correct_hash_provided: - ttl = self.ds.config("default_cache_ttl_hashed") + ttl = self.ds.setting("default_cache_ttl_hashed") else: - ttl = self.ds.config("default_cache_ttl") + ttl = self.ds.setting("default_cache_ttl") return self.set_response_headers(r, ttl) diff --git a/datasette/views/database.py b/datasette/views/database.py index d4ed8570..17c78150 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -136,7 +136,7 @@ class DatabaseView(DataView): "show_hidden": request.args.get("_show_hidden"), "editable": True, "metadata": metadata, - "allow_download": self.ds.config("allow_download") + "allow_download": self.ds.setting("allow_download") and not db.is_mutable and database != ":memory:", }, @@ -161,7 +161,7 @@ class DatabaseDownload(DataView): db = self.ds.databases[database] if db.is_memory: raise DatasetteError("Cannot download :memory: database", status=404) - if not self.ds.config("allow_download") or db.is_mutable: + if not self.ds.setting("allow_download") or db.is_mutable: raise Forbidden("Database download is forbidden") if not db.path: raise DatasetteError("Cannot download database", status=404) diff --git a/datasette/views/table.py b/datasette/views/table.py index 09c2d740..a0de2a8e 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -121,7 +121,7 @@ class RowTableShared(DataView): } cell_rows = [] - base_url = self.ds.config("base_url") + base_url = self.ds.setting("base_url") for row in rows: cells = [] # Unless we are a view, the first column is a link - either to the rowid @@ -654,7 +654,7 @@ class TableView(RowTableShared): pass # facets support - if not self.ds.config("allow_facet") and any( + if not self.ds.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") @@ -772,8 +772,8 @@ class TableView(RowTableShared): suggested_facets = [] if ( - self.ds.config("suggest_facets") - and self.ds.config("allow_facet") + self.ds.setting("suggest_facets") + and self.ds.setting("allow_facet") and not _next ): for facet in facet_instances: @@ -801,7 +801,7 @@ class TableView(RowTableShared): results.description, rows, link_column=not is_view, - truncate_cells=self.ds.config("truncate_cells_html"), + truncate_cells=self.ds.setting("truncate_cells_html"), ) metadata = ( (self.ds.metadata("databases") or {}) diff --git a/docs/internals.rst b/docs/internals.rst index cec1268f..78d4e5d2 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -350,7 +350,21 @@ Returns the absolute URL for the given path, including the protocol and host. Fo absolute_url = datasette.absolute_url(request, "/dbname/table.json") # Would return "http://localhost:8001/dbname/table.json" -The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`settings_force_https_urls` configuration setting is taken into account. +The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`setting_force_https_urls` configuration setting is taken into account. + +.setting(key) +------------- + +``key`` - string + The name of the setting, e.g. ``base_url``. + +Returns the configured value for the specified :ref:`setting `. This can be a string, boolean or integer depending on the requested setting. + +For example: + +.. code-block:: python + + downloads_are_allowed = datasette.setting("allow_download") .. _internals_datasette_client: diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 0be0b932..56bc2fb4 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -33,3 +33,15 @@ def test_sign_unsign(datasette, value, namespace): assert value == datasette.unsign(signed, *extra_args) with pytest.raises(BadSignature): datasette.unsign(signed[:-1] + ("!" if signed[-1] != "!" else ":")) + + +@pytest.mark.parametrize( + "setting,expected", + ( + ("base_url", "/"), + ("max_csv_mb", 100), + ("allow_csv_stream", True), + ), +) +def test_datasette_setting(datasette, setting, expected): + assert datasette.setting(setting) == expected diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 0b1c5f0e..c538bef1 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -33,10 +33,10 @@ async def test_client_methods(datasette, method, path, expected_status): @pytest.mark.asyncio @pytest.mark.parametrize("prefix", [None, "/prefix/"]) async def test_client_post(datasette, prefix): - original_base_url = datasette._config["base_url"] + original_base_url = datasette._settings["base_url"] try: if prefix is not None: - datasette._config["base_url"] = prefix + datasette._settings["base_url"] = prefix response = await datasette.client.post( "/-/messages", data={ @@ -48,7 +48,7 @@ async def test_client_post(datasette, prefix): assert response.status_code == 302 assert "ds_messages" in response.cookies finally: - datasette._config["base_url"] = original_base_url + datasette._settings["base_url"] = original_base_url @pytest.mark.asyncio @@ -56,12 +56,12 @@ async def test_client_post(datasette, prefix): "prefix,expected_path", [(None, "/asgi-scope"), ("/prefix/", "/prefix/asgi-scope")] ) async def test_client_path(datasette, prefix, expected_path): - original_base_url = datasette._config["base_url"] + original_base_url = datasette._settings["base_url"] try: if prefix is not None: - datasette._config["base_url"] = prefix + datasette._settings["base_url"] = prefix response = await datasette.client.get("/asgi-scope") path = response.json()["path"] assert path == expected_path finally: - datasette._config["base_url"] = original_base_url + datasette._settings["base_url"] = original_base_url diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index 89290911..fd05c1b6 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -20,14 +20,14 @@ def ds(): ], ) def test_path(ds, base_url, path, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.path(path) assert actual == expected assert isinstance(actual, PrefixedUrlString) def test_path_applied_twice_does_not_double_prefix(ds): - ds._config["base_url"] = "/prefix/" + ds._settings["base_url"] = "/prefix/" path = ds.urls.path("/") assert path == "/prefix/" path = ds.urls.path(path) @@ -42,7 +42,7 @@ def test_path_applied_twice_does_not_double_prefix(ds): ], ) def test_instance(ds, base_url, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.instance() assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -56,7 +56,7 @@ def test_instance(ds, base_url, expected): ], ) def test_static(ds, base_url, file, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.static(file) assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -80,7 +80,7 @@ def test_static(ds, base_url, file, expected): ], ) def test_static_plugins(ds, base_url, plugin, file, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.static_plugins(plugin, file) assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -94,7 +94,7 @@ def test_static_plugins(ds, base_url, plugin, file, expected): ], ) def test_logout(ds, base_url, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.logout() assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -109,7 +109,7 @@ def test_logout(ds, base_url, expected): ], ) def test_database(ds, base_url, format, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.database(":memory:", format=format) assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -125,7 +125,7 @@ def test_database(ds, base_url, format, expected): ], ) def test_table_and_query(ds, base_url, name, format, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual1 = ds.urls.table(":memory:", name, format=format) assert actual1 == expected assert isinstance(actual1, PrefixedUrlString) @@ -143,7 +143,7 @@ def test_table_and_query(ds, base_url, name, format, expected): ], ) def test_row(ds, base_url, format, expected): - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url actual = ds.urls.row(":memory:", "facetable", "1", format=format) assert actual == expected assert isinstance(actual, PrefixedUrlString) @@ -152,9 +152,9 @@ def test_row(ds, base_url, format, expected): @pytest.mark.parametrize("base_url", ["/", "/prefix/"]) def test_database_hashed(app_client_with_hash, base_url): ds = app_client_with_hash.ds - original_base_url = ds._config["base_url"] + original_base_url = ds._settings["base_url"] try: - ds._config["base_url"] = base_url + ds._settings["base_url"] = base_url db_hash = ds.get_database("fixtures").hash assert len(db_hash) == 64 expected = f"{base_url}fixtures-{db_hash[:7]}" @@ -163,4 +163,4 @@ def test_database_hashed(app_client_with_hash, base_url): assert ds.urls.query("fixtures", "name") == expected + "/name" finally: # Reset this since fixture is shared with other tests - ds._config["base_url"] = original_base_url + ds._settings["base_url"] = original_base_url From 37d18a5bce08c9ee53c080f613bae84fc2ccc853 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 24 Nov 2020 19:05:35 -0800 Subject: [PATCH 0174/1866] datasette publish cloudrun --apt-get-install, closes #1110 --- datasette/publish/cloudrun.py | 8 +++ datasette/utils/__init__.py | 27 +++++--- docs/datasette-publish-cloudrun-help.txt | 1 + tests/test_publish_cloudrun.py | 78 +++++++++++++++++++++--- 4 files changed, 97 insertions(+), 17 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 54f55fcb..54f06da0 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -36,6 +36,12 @@ def publish_subcommand(publish): callback=_validate_memory, help="Memory to allocate in Cloud Run, e.g. 1Gi", ) + @click.option( + "--apt-get-install", + "apt_get_extras", + multiple=True, + help="Additional packages to apt-get install", + ) def cloudrun( files, metadata, @@ -60,6 +66,7 @@ def publish_subcommand(publish): spatialite, show_files, memory, + apt_get_extras, ): fail_if_publish_binary_not_installed( "gcloud", "Google Cloud", "https://cloud.google.com/sdk/" @@ -122,6 +129,7 @@ def publish_subcommand(publish): secret, extra_metadata, environment_variables, + apt_get_extras=apt_get_extras, ): if show_files: if os.path.exists("metadata.json"): diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d62302e9..54a5b247 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -47,11 +47,10 @@ reserved_words = set( ).split() ) -SPATIALITE_DOCKERFILE_EXTRAS = r""" +APT_GET_DOCKERFILE_EXTRAS = r""" RUN apt-get update && \ - apt-get install -y python3-dev gcc libsqlite3-mod-spatialite && \ + apt-get install -y {} && \ rm -rf /var/lib/apt/lists/* -ENV SQLITE_EXTENSIONS /usr/lib/x86_64-linux-gnu/mod_spatialite.so """ # Can replace with sqlite-utils when I add that dependency @@ -308,10 +307,12 @@ def make_dockerfile( secret, environment_variables=None, port=8001, + apt_get_extras=None, ): cmd = ["datasette", "serve", "--host", "0.0.0.0"] environment_variables = environment_variables or {} environment_variables["DATASETTE_SECRET"] = secret + apt_get_extras = apt_get_extras or [] for filename in files: cmd.extend(["-i", filename]) cmd.extend(["--cors", "--inspect-file", "inspect-data.json"]) @@ -340,28 +341,38 @@ def make_dockerfile( else: install = ["datasette"] + list(install) + apt_get_extras_ = [] + apt_get_extras_.extend(apt_get_extras) + apt_get_extras = apt_get_extras_ + if spatialite: + apt_get_extras.extend(["python3-dev", "gcc", "libsqlite3-mod-spatialite"]) + environment_variables[ + "SQLITE_EXTENSIONS" + ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" return """ FROM python:3.8 COPY . /app WORKDIR /app -{spatialite_extras} +{apt_get_extras} {environment_variables} RUN pip install -U {install_from} RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT {port} EXPOSE {port} CMD {cmd}""".format( + apt_get_extras=APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) + if apt_get_extras + else "", environment_variables="\n".join( [ "ENV {} '{}'".format(key, value) for key, value in environment_variables.items() ] ), - files=" ".join(files), - cmd=cmd, install_from=" ".join(install), - spatialite_extras=SPATIALITE_DOCKERFILE_EXTRAS if spatialite else "", + files=" ".join(files), port=port, + cmd=cmd, ).strip() @@ -382,6 +393,7 @@ def temporary_docker_directory( extra_metadata=None, environment_variables=None, port=8001, + apt_get_extras=None, ): extra_metadata = extra_metadata or {} tmp = tempfile.TemporaryDirectory() @@ -415,6 +427,7 @@ def temporary_docker_directory( secret, environment_variables, port=port, + apt_get_extras=apt_get_extras, ) os.chdir(datasette_dir) if metadata_content: diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index a625bd10..8cf293d9 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -30,4 +30,5 @@ Options: --spatialite Enable SpatialLite extension --show-files Output the generated Dockerfile and metadata.json --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi + --apt-get-install TEXT Additional packages to apt-get install --help Show this message and exit. diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index e629bba0..7adef39d 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -182,22 +182,26 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): "x-secret", ], ) + assert result.exit_code == 0 dockerfile = ( result.output.split("==== Dockerfile ====\n")[1] .split("\n====================\n")[0] .strip() ) - expected = """FROM python:3.8 -COPY . /app -WORKDIR /app + expected = textwrap.dedent( + r""" + FROM python:3.8 + COPY . /app + WORKDIR /app -ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' -ENV DATASETTE_SECRET 'x-secret' -RUN pip install -U datasette -RUN datasette inspect test.db --inspect-file inspect-data.json -ENV PORT 8001 -EXPOSE 8001 -CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --port $PORT""".strip() + ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' + ENV DATASETTE_SECRET 'x-secret' + RUN pip install -U datasette + RUN datasette inspect test.db --inspect-file inspect-data.json + ENV PORT 8001 + EXPOSE 8001 + CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --port $PORT""" + ).strip() assert expected == dockerfile metadata = ( result.output.split("=== metadata.json ===\n")[1] @@ -213,3 +217,57 @@ CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data } }, } == json.loads(metadata) + + +@mock.patch("shutil.which") +@mock.patch("datasette.publish.cloudrun.check_output") +@mock.patch("datasette.publish.cloudrun.check_call") +def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): + mock_which.return_value = True + mock_output.return_value = "myproject" + + runner = CliRunner() + with runner.isolated_filesystem(): + open("test.db", "w").write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--service", + "datasette", + "--show-files", + "--secret", + "x-secret", + "--apt-get-install", + "ripgrep", + "--spatialite", + ], + ) + assert result.exit_code == 0 + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = textwrap.dedent( + r""" + FROM python:3.8 + COPY . /app + WORKDIR /app + + RUN apt-get update && \ + apt-get install -y ripgrep python3-dev gcc libsqlite3-mod-spatialite && \ + rm -rf /var/lib/apt/lists/* + + ENV DATASETTE_SECRET 'x-secret' + ENV SQLITE_EXTENSIONS '/usr/lib/x86_64-linux-gnu/mod_spatialite.so' + RUN pip install -U datasette + RUN datasette inspect test.db --inspect-file inspect-data.json + ENV PORT 8001 + EXPOSE 8001 + CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --port $PORT + """ + ).strip() + assert expected == dockerfile From bbde835a1fec01458e8d00929e7bab6d6a5ba948 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Sat, 28 Nov 2020 13:53:48 -0600 Subject: [PATCH 0175/1866] Fix --metadata doc usage (#1112) Thanks, @jefftriplett. --- docs/changelog.rst | 4 ++-- docs/settings.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2916b373..20181ca9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -453,7 +453,7 @@ You can now create :ref:`custom pages ` within your Datasette inst :ref:`settings_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: $ datasette one.db two.db \ - --metadata.json \ + --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ --static css:css @@ -770,7 +770,7 @@ Small changes 0.28 (2019-05-19) ----------------- -A `salmagundi `__ of new features! +A `salmagundi `__ of new features! .. _v0_28_databases_that_change: diff --git a/docs/settings.rst b/docs/settings.rst index 350fd048..156893e0 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -23,7 +23,7 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: $ datasette one.db two.db \ - --metadata.json \ + --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ --static css:css From 50cc6af01672526791900df7c8834a62fa094852 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 28 Nov 2020 15:34:56 -0800 Subject: [PATCH 0176/1866] Fixed some broken internal links, refs #1106 --- docs/changelog.rst | 14 +++++++------- docs/csv_export.rst | 6 +++--- docs/deploying.rst | 2 +- docs/pages.rst | 2 +- docs/performance.rst | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 20181ca9..15992020 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -353,9 +353,9 @@ Signed values and secrets Both flash messages and user authentication needed a way to sign values and set signed cookies. Two new methods are now available for plugins to take advantage of this mechanism: :ref:`datasette_sign` and :ref:`datasette_unsign`. -Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`settings_secret` for more details. +Datasette will generate a secret automatically when it starts up, but to avoid resetting the secret (and hence invalidating any cookies) every time the server restarts you should set your own secret. You can pass a secret to Datasette using the new ``--secret`` option or with a ``DATASETTE_SECRET`` environment variable. See :ref:`setting_secret` for more details. -You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`settings_publish_secrets`. +You can also set a secret when you deploy Datasette using ``datasette publish`` or ``datasette package`` - see :ref:`setting_publish_secrets`. Plugins can now sign values and verify their signatures using the :ref:`datasette.sign() ` and :ref:`datasette.unsign() ` methods. @@ -450,7 +450,7 @@ A small release which provides improved internal methods for use in plugins, alo You can now create :ref:`custom pages ` within your Datasette instance using a custom template file. For example, adding a template file called ``templates/pages/about.html`` will result in a new page being served at ``/about`` on your instance. See the :ref:`custom pages documentation ` for full details, including how to return custom HTTP headers, redirects and status codes. (`#648 `__) -:ref:`settings_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: +:ref:`config_dir` (`#731 `__) allows you to define a custom Datasette instance as a directory. So instead of running the following:: $ datasette one.db two.db \ --metadata=metadata.json \ @@ -480,7 +480,7 @@ Also in this release: * Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. See :ref:`metadata_yaml`. (`#713 `__) * Removed support for ``datasette publish now``, which used the the now-retired Zeit Now v1 hosting platform. A new plugin, `datasette-publish-now `__, can be installed to publish data to Zeit (`now Vercel `__) Now v2. (`#710 `__) * Fixed a bug where the ``extra_template_vars(request, view_name)`` plugin hook was not receiving the correct ``view_name``. (`#716 `__) -* Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`settings_template_debug`). (`#693 `__) +* Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`setting_template_debug`). (`#693 `__) * Fixed a bug where the "templates considered" HTML comment was no longer being displayed. (`#689 `__) * Fixed a ``datasette publish`` bug where ``--plugin-secret`` would over-ride plugin configuration in the provided ``metadata.json`` file. (`#724 `__) * Added a new CSS class for customizing the canned query page. (`#727 `__) @@ -565,7 +565,7 @@ Also in this release: * asyncio task information is now included on the ``/-/threads`` debug page * Bumped Uvicorn dependency 0.11 * You can now use ``--port 0`` to listen on an available port -* New :ref:`settings_template_debug` setting for debugging templates, e.g. https://latest.datasette.io/fixtures/roadside_attractions?_context=1 (`#654 `__) +* New :ref:`setting_template_debug` setting for debugging templates, e.g. https://latest.datasette.io/fixtures/roadside_attractions?_context=1 (`#654 `__) .. _v0_32: @@ -941,7 +941,7 @@ A number of small new features: - ``datasette publish heroku`` now supports ``--extra-options``, fixes `#334 `_ - Custom error message if SpatiaLite is needed for specified database, closes `#331 `_ -- New config option: ``truncate_cells_html`` for :ref:`truncating long cell values ` in HTML view - closes `#330 `_ +- New config option: ``truncate_cells_html`` for :ref:`truncating long cell values ` in HTML view - closes `#330 `_ - Documentation for :ref:`datasette publish and datasette package `, closes `#337 `_ - Fixed compatibility with Python 3.7 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett] @@ -1000,7 +1000,7 @@ Check out the :ref:`CSV export documentation ` for more details, or try the feature out on https://fivethirtyeight.datasettes.com/fivethirtyeight/bechdel%2Fmovies -If your table has more than :ref:`settings_max_returned_rows` (default 1,000) +If your table has more than :ref:`setting_max_returned_rows` (default 1,000) Datasette provides the option to *stream all rows*. This option takes advantage of async Python and Datasette's efficient :ref:`pagination ` to iterate through the entire matching result set and stream it back as a diff --git a/docs/csv_export.rst b/docs/csv_export.rst index 704cc19d..0bda20ef 100644 --- a/docs/csv_export.rst +++ b/docs/csv_export.rst @@ -23,7 +23,7 @@ file, which looks like this and has the following options: the ``city_id`` column is accompanied by a ``city_id_label`` column. * **stream all rows** - by default CSV files only contain the first - :ref:`settings_max_returned_rows` records. This option will cause Datasette to + :ref:`setting_max_returned_rows` records. This option will cause Datasette to loop through every matching record and return them as a single CSV file. You can try that out on https://latest.datasette.io/fixtures/facetable?_size=4 @@ -40,9 +40,9 @@ Since databases can get pretty large, by default this option is capped at 100MB if a table returns more than 100MB of data the last line of the CSV will be a truncation error message. -You can increase or remove this limit using the :ref:`settings_max_csv_mb` config +You can increase or remove this limit using the :ref:`setting_max_csv_mb` config setting. You can also disable the CSV export feature entirely using -:ref:`settings_allow_csv_stream`. +:ref:`setting_allow_csv_stream`. A note on URLs -------------- diff --git a/docs/deploying.rst b/docs/deploying.rst index 4ca0e82a..d1abe6a3 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -58,7 +58,7 @@ Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Data $ python3 -c 'import secrets; print(secrets.token_hex(32))' -This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`settings_dir` for details. +This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. You can start the Datasette process running using the following:: diff --git a/docs/pages.rst b/docs/pages.rst index 5f77bec7..0941c960 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -66,7 +66,7 @@ Row Every row in every Datasette table has its own URL. This means individual records can be linked to directly. -Table cells with extremely long text contents are truncated on the table view according to the :ref:`settings_truncate_cells_html` setting. If a cell has been truncated the full length version of that cell will be available on the row page. +Table cells with extremely long text contents are truncated on the table view according to the :ref:`setting_truncate_cells_html` setting. If a cell has been truncated the full length version of that cell will be available on the row page. Rows which are the targets of foreign key references from other tables will show a link to a filtered search for all records that reference that row. Here's an example from the Registers of Members Interests database: diff --git a/docs/performance.rst b/docs/performance.rst index 1d24adce..2727416d 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -56,7 +56,7 @@ Using a caching proxy in this way could enable a Datasette-backed visualization Datasette's integration with HTTP caches can be enabled using a combination of configuration options and querystring arguments. -The :ref:`settings_default_cache_ttl` setting sets the default HTTP cache TTL for all Datasette pages. This is 5 seconds unless you change it - you can set it to 0 if you wish to disable HTTP caching entirely. +The :ref:`setting_default_cache_ttl` setting sets the default HTTP cache TTL for all Datasette pages. This is 5 seconds unless you change it - you can set it to 0 if you wish to disable HTTP caching entirely. You can also change the cache timeout on a per-request basis using the ``?_ttl=10`` querystring parameter. This can be useful when you are working with the Datasette JSON API - you may decide that a specific query can be cached for a longer time, or maybe you need to set ``?_ttl=0`` for some requests for example if you are running a SQL ``order by random()`` query. @@ -65,9 +65,9 @@ Hashed URL mode When you open a database file in immutable mode using the ``-i`` option, Datasette calculates a SHA-256 hash of the contents of that file on startup. This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future. This results in URLs that can then be cached indefinitely by both browsers and caching proxies - an enormous potential performance optimization. -You can enable these hashed URLs in two ways: using the :ref:`settings_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` querystring parameter (which only applies to the current request). +You can enable these hashed URLs in two ways: using the :ref:`setting_hash_urls` configuration setting (which affects all requests to Datasette) or via the ``?_hash=1`` querystring parameter (which only applies to the current request). -With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`settings_default_cache_ttl_hashed` which defaults to 365 days. +With hashed URLs enabled, any request to e.g. ``/mydatabase/mytable`` will 302 redirect to ``mydatabase-455fe3a/mytable``. The URL containing the hash will be served with a very long cache expire header - configured using :ref:`setting_default_cache_ttl_hashed` which defaults to 365 days. Since these responses are cached for a long time, you may wish to build API clients against the non-hashed version of these URLs. These 302 redirects are served extremely quickly, so this should still be a performant way to work against the Datasette API. From a8e66f9065fb55a3863cc05dfb2ce52f9618cdb7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 28 Nov 2020 15:54:35 -0800 Subject: [PATCH 0177/1866] Release 0.52 Refs #992, #1103, #1104, #1107, #1077, #1110, #1089, #1086, #1088, #1084 --- README.md | 1 + datasette/version.py | 2 +- docs/changelog.rst | 24 ++++++++++++++++++++++++ docs/internals.rst | 2 ++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a10ccfd3..c0019e9b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly new ## News + * 28th November 2020: [Datasette 0.52](https://docs.datasette.io/en/stable/changelog.html#v0-52) - `--config` is now `--setting`, new `database_actions` plugin hook, `datasette publish cloudrun --apt-get-install` option and several bug fixes. * 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. [Annotated release notes](https://simonwillison.net/2020/Nov/1/datasette-0-51/). * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). diff --git a/datasette/version.py b/datasette/version.py index 2d949370..3b84c97b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.51.1" +__version__ = "0.52" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 15992020..49772638 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,30 @@ Changelog ========= +.. _v0_52: + +0.52 (2020-11-28) +----------------- + +This release includes a number of changes relating to an internal rebranding effort: Datasette's **configuration** mechanism (things like ``datasette --config default_page_size:10``) has been renamed to **settings**. + +- New ``--setting default_page_size 10`` option as a replacement for ``--config default_page_size:10`` (note the lack of a colon). The ``--config`` option is deprecated but will continue working until Datasette 1.0. (`#992 `__) +- The ``/-/config`` introspection page is now ``/-/settings``, and the previous page redirects to the new one. (`#1103 `__) +- The ``config.json`` file in :ref:`config_dir` is now called ``settings.json``. (`#1104 `__) +- The undocumented ``datasette.config()`` internal method has been replaced by a documented :ref:`datasette_setting` method. (`#1107 `__) + +Also in this release: + +- New plugin hook: :ref:`plugin_hook_database_actions`, which adds menu items to a new cog menu shown at the top of the database page. (`#1077 `__) +- ``datasette publish cloudrun`` has a new ``--apt-get-install`` option that can be used to install additional Ubuntu packages as part of the deployment. This is useful for deploying the new `datasette-ripgrep plugin `__. (`#1110 `__) +- Swept the documentation to remove words that minimize involved difficulty. (`#1089 `__) + +And some bug fixes: + +- Foreign keys linking to rows with blank label columns now display as a hyphen, allowing those links to be clicked. (`#1086 `__) +- Fixed bug where row pages could sometimes 500 if the underlying queries exceeded a time limit. (`#1088 `__) +- Fixed a bug where the table action menu could appear partially obscured by the edge of the page. (`#1084 `__) + .. _v0_51_1: 0.51.1 (2020-10-31) diff --git a/docs/internals.rst b/docs/internals.rst index 78d4e5d2..ff566f69 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -352,6 +352,8 @@ Returns the absolute URL for the given path, including the protocol and host. Fo The current request object is used to determine the hostname and protocol that should be used for the returned URL. The :ref:`setting_force_https_urls` configuration setting is taken into account. +.. _datasette_setting: + .setting(key) ------------- From 12877d7a48e2aa28bb5e780f929a218f7265d849 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 28 Nov 2020 23:44:57 -0800 Subject: [PATCH 0178/1866] Plugin testing docs now recommend datasette.client, closes #1102 --- docs/testing_plugins.rst | 57 +++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 76f69a6a..d8ebdc77 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -11,24 +11,24 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl from datasette.app import Datasette import pytest - import httpx + @pytest.mark.asyncio async def test_plugin_is_installed(): - app = Datasette([], memory=True).app() - async with httpx.AsyncClient(app=app) as client: - response = await client.get("http://localhost/-/plugins.json") - assert 200 == response.status_code - installed_plugins = {p["name"] for p in response.json()} - assert "datasette-plugin-template-demo" in installed_plugins + datasette = Datasette([], memory=True) + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + installed_plugins = {p["name"] for p in response.json()} + assert "datasette-plugin-template-demo" in installed_plugins -This test uses the `HTTPX `__ Python library to run mock HTTP requests through a fresh instance of Datasette. This is the recommended way to write tests against a Datasette instance. -It also uses the `pytest-asyncio `__ package to add support for ``async def`` test functions running under pytest. +This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX `__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. + +This test also uses the `pytest-asyncio `__ package to add support for ``async def`` test functions running under pytest. You can install these packages like so:: - pip install pytest pytest-asyncio httpx + pip install pytest pytest-asyncio If you are building an installable package you can add them as test dependencies to your ``setup.py`` module like this: @@ -38,7 +38,7 @@ If you are building an installable package you can add them as test dependencies name="datasette-my-plugin", # ... extras_require={ - "test": ["pytest", "pytest-asyncio", "httpx"] + "test": ["pytest", "pytest-asyncio"] }, tests_require=["datasette-my-plugin[test]"], ) @@ -65,12 +65,11 @@ Here's an example that uses the `sqlite-utils library Some dogs" in response.text + async def test_example_table_html(datasette): + response = await datasette.client.get("/test/dogs") + assert ">Some dogs" in response.text -Here the ``ds()`` function defines the fixture, which is than automatically passed to the two test functions based on pytest automatically matching their ``ds`` function parameters. +Here the ``datasette()`` function defines the fixture, which is than automatically passed to the two test functions based on pytest automatically matching their ``datasette`` function parameters. The ``@pytest.fixture(scope="session")`` line here ensures the fixture is reused for the full ``pytest`` execution session. This means that the temporary database file will be created once and reused for each test. @@ -119,5 +116,5 @@ If you want to create that test database repeatedly for every individual test fu .. code-block:: python @pytest.fixture - def ds(tmp_path_factory): - # ... + def datasette(tmp_path_factory): + # This fixture will be executed repeatedly for every test From e800ffcf7cc6a915eb554b369c654f87162575e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 09:37:43 -0800 Subject: [PATCH 0179/1866] /usr/local/lib/mod_spatialite.so Closes #1114 --- datasette/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 54a5b247..d326c773 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -57,6 +57,7 @@ RUN apt-get update && \ SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", "/usr/local/lib/mod_spatialite.dylib", + "/usr/local/lib/mod_spatialite.so", ) # Length of hash subset used in hashed URLs: HASH_LENGTH = 7 From deb0be4ae56f191f121239b29e83dd53b62d6305 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 11:30:17 -0800 Subject: [PATCH 0180/1866] Fix bug where compound foreign keys produced broken links, closes #1098 --- datasette/utils/__init__.py | 51 +++++++++++++++++++++----------- tests/fixtures.py | 7 +++-- tests/test_api.py | 14 +++++++-- tests/test_csv.py | 6 ++-- tests/test_html.py | 6 ++++ tests/test_internals_database.py | 33 ++++++++++++++++++--- 6 files changed, 88 insertions(+), 29 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d326c773..d467383d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,7 +1,7 @@ import asyncio from contextlib import contextmanager import click -from collections import OrderedDict, namedtuple +from collections import OrderedDict, namedtuple, Counter import base64 import hashlib import inspect @@ -474,9 +474,25 @@ def get_outbound_foreign_keys(conn, table): if info is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = info fks.append( - {"column": from_, "other_table": table_name, "other_column": to_} + { + "column": from_, + "other_table": table_name, + "other_column": to_, + "id": id, + "seq": seq, + } ) - return fks + # Filter out compound foreign keys by removing any where "id" is not unique + id_counts = Counter(fk["id"] for fk in fks) + return [ + { + "column": fk["column"], + "other_table": fk["other_table"], + "other_column": fk["other_column"], + } + for fk in fks + if id_counts[fk["id"]] == 1 + ] def get_all_foreign_keys(conn): @@ -487,20 +503,21 @@ def get_all_foreign_keys(conn): for table in tables: table_to_foreign_keys[table] = {"incoming": [], "outgoing": []} for table in tables: - infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall() - for info in infos: - if info is not None: - id, seq, table_name, from_, to_, on_update, on_delete, match = info - if table_name not in table_to_foreign_keys: - # Weird edge case where something refers to a table that does - # not actually exist - continue - table_to_foreign_keys[table_name]["incoming"].append( - {"other_table": table, "column": to_, "other_column": from_} - ) - table_to_foreign_keys[table]["outgoing"].append( - {"other_table": table_name, "column": from_, "other_column": to_} - ) + fks = get_outbound_foreign_keys(conn, table) + for fk in fks: + table_name = fk["other_table"] + from_ = fk["column"] + to_ = fk["other_column"] + if table_name not in table_to_foreign_keys: + # Weird edge case where something refers to a table that does + # not actually exist + continue + table_to_foreign_keys[table_name]["incoming"].append( + {"other_table": table, "column": to_, "other_column": from_} + ) + table_to_foreign_keys[table]["outgoing"].append( + {"other_table": table_name, "column": from_, "other_column": to_} + ) return table_to_foreign_keys diff --git a/tests/fixtures.py b/tests/fixtures.py index 3abca821..f95a2d6b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -388,9 +388,12 @@ CREATE TABLE foreign_key_references ( foreign_key_with_label varchar(30), foreign_key_with_blank_label varchar(30), foreign_key_with_no_label varchar(30), + foreign_key_compound_pk1 varchar(30), + foreign_key_compound_pk2 varchar(30), FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) + FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) ); CREATE TABLE sortable ( @@ -624,8 +627,8 @@ INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1); -INSERT INTO foreign_key_references VALUES (2, null, null, null); +INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); +INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); INSERT INTO custom_foreign_key_label VALUES (1, 1); diff --git a/tests/test_api.py b/tests/test_api.py index 2bab6c30..848daf9c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -237,6 +237,8 @@ def test_database_page(app_client): "foreign_key_with_label", "foreign_key_with_blank_label", "foreign_key_with_no_label", + "foreign_key_compound_pk1", + "foreign_key_compound_pk2", ], "primary_keys": ["pk"], "count": 2, @@ -1637,6 +1639,8 @@ def test_expand_label(app_client): "foreign_key_with_label": {"value": "1", "label": "hello"}, "foreign_key_with_blank_label": "3", "foreign_key_with_no_label": "1", + "foreign_key_compound_pk1": "a", + "foreign_key_compound_pk2": "b", } } @@ -1821,24 +1825,28 @@ def test_common_prefix_database_names(app_client_conflicting_database_names): assert db_name == data["database"] -def test_null_foreign_keys_are_not_expanded(app_client): +def test_null_and_compound_foreign_keys_are_not_expanded(app_client): response = app_client.get( "/fixtures/foreign_key_references.json?_shape=array&_labels=on" ) - assert [ + assert response.json == [ { "pk": "1", "foreign_key_with_label": {"value": "1", "label": "hello"}, "foreign_key_with_blank_label": {"value": "3", "label": ""}, "foreign_key_with_no_label": {"value": "1", "label": "1"}, + "foreign_key_compound_pk1": "a", + "foreign_key_compound_pk2": "b", }, { "pk": "2", "foreign_key_with_label": None, "foreign_key_with_blank_label": None, "foreign_key_with_no_label": None, + "foreign_key_compound_pk1": None, + "foreign_key_compound_pk2": None, }, - ] == response.json + ] def test_inspect_file_used_for_count(app_client_immutable_and_inspect_file): diff --git a/tests/test_csv.py b/tests/test_csv.py index 209bce2b..0fd665a9 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -42,9 +42,9 @@ pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,com ) EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV = """ -pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_blank_label,foreign_key_with_blank_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label -1,1,hello,3,,1,1 -2,,,,,, +pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_blank_label,foreign_key_with_blank_label_label,foreign_key_with_no_label,foreign_key_with_no_label_label,foreign_key_compound_pk1,foreign_key_compound_pk2 +1,1,hello,3,,1,1,a,b +2,,,,,,,, """.lstrip().replace( "\n", "\r\n" ) diff --git a/tests/test_html.py b/tests/test_html.py index d53dbabc..ecbf89b4 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -804,12 +804,16 @@ def test_table_html_foreign_key_links(app_client): '    ', '', '', + '', + '', ], [ '', '', '', '', + '', + '', ], ] @@ -836,6 +840,8 @@ def test_table_html_disable_foreign_key_links_with_labels(app_client): '', '', '', + '', + '', ] ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e5938f3b..7c8f478c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -267,7 +267,7 @@ async def test_table_column_details(db, table, expected): @pytest.mark.asyncio async def test_get_all_foreign_keys(db): all_foreign_keys = await db.get_all_foreign_keys() - assert { + assert all_foreign_keys["roadside_attraction_characteristics"] == { "incoming": [], "outgoing": [ { @@ -281,8 +281,8 @@ async def test_get_all_foreign_keys(db): "other_column": "pk", }, ], - } == all_foreign_keys["roadside_attraction_characteristics"] - assert { + } + assert all_foreign_keys["attraction_characteristic"] == { "incoming": [ { "other_table": "roadside_attraction_characteristics", @@ -291,7 +291,32 @@ async def test_get_all_foreign_keys(db): } ], "outgoing": [], - } == all_foreign_keys["attraction_characteristic"] + } + assert all_foreign_keys["compound_primary_key"] == { + # No incoming because these are compound foreign keys, which we currently ignore + "incoming": [], + "outgoing": [], + } + assert all_foreign_keys["foreign_key_references"] == { + "incoming": [], + "outgoing": [ + { + "other_table": "primary_key_multiple_columns", + "column": "foreign_key_with_no_label", + "other_column": "id", + }, + { + "other_table": "simple_primary_key", + "column": "foreign_key_with_blank_label", + "other_column": "id", + }, + { + "other_table": "simple_primary_key", + "column": "foreign_key_with_label", + "other_column": "id", + }, + ], + } @pytest.mark.asyncio From 242bc89fdf2e775e340d69a4e851b3a9accb31c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 11:38:29 -0800 Subject: [PATCH 0181/1866] Release 0.52.1 Refs #1098, #1102, #1114 --- datasette/version.py | 2 +- docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 3b84c97b..119295b3 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52" +__version__ = "0.52.1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 49772638..a77cf5a5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_52_1: + +0.52.1 (2020-11-29) +------------------- + +- Documentation on :ref:`testing_plugins` now recommends using :ref:`internals_datasette_client`. (`#1102 `__) +- Fix bug where compound foreign keys produced broken links. (`#1098 `__) +- ``datasette --load-module=spatialite`` now also checks for ``/usr/local/lib/mod_spatialite.so``. Thanks, Dan Peterson. (`#1114 `__) + .. _v0_52: 0.52 (2020-11-28) From 09033c08bec8555e0e893e077afa10a7a75d7d35 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 12:13:16 -0800 Subject: [PATCH 0182/1866] Suggest --load-extension=spatialite, closes #1115 --- datasette/cli.py | 12 ++++++++++-- tests/test_cli.py | 29 ++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 5feab51e..e84695e3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -16,6 +16,7 @@ from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, pm from .utils import ( StartupError, check_connection, + find_spatialite, parse_metadata, ConnectionProblem, SpatialiteConnectionProblem, @@ -537,10 +538,17 @@ async def check_databases(ds): try: await database.execute_fn(check_connection) except SpatialiteConnectionProblem: + suggestion = "" + try: + find_spatialite() + suggestion = "\n\nTry adding the --load-extension=spatialite option." + except SpatialiteNotFound: + pass raise click.UsageError( "It looks like you're trying to load a SpatiaLite" - " database without first loading the SpatiaLite module." - "\n\nRead more: https://docs.datasette.io/en/stable/spatialite.html" + + " database without first loading the SpatiaLite module." + + suggestion + + "\n\nRead more: https://docs.datasette.io/en/stable/spatialite.html" ) except ConnectionProblem as e: raise click.UsageError( diff --git a/tests/test_cli.py b/tests/test_cli.py index 36b9a092..409408ae 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -59,13 +59,28 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): assert {"hithere": 44} == db.cached_table_counts -def test_spatialite_error_if_attempt_to_open_spatialite(): - runner = CliRunner() - result = runner.invoke( - cli, ["serve", str(pathlib.Path(__file__).parent / "spatialite.db")] - ) - assert result.exit_code != 0 - assert "trying to load a SpatiaLite database" in result.output +@pytest.mark.parametrize( + "spatialite_paths,should_suggest_load_extension", + ( + ([], False), + (["/tmp"], True), + ), +) +def test_spatialite_error_if_attempt_to_open_spatialite( + spatialite_paths, should_suggest_load_extension +): + with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths): + runner = CliRunner() + result = runner.invoke( + cli, ["serve", str(pathlib.Path(__file__).parent / "spatialite.db")] + ) + assert result.exit_code != 0 + assert "It looks like you're trying to load a SpatiaLite" in result.output + suggestion = "--load-extension=spatialite" + if should_suggest_load_extension: + assert suggestion in result.output + else: + assert suggestion not in result.output @mock.patch("datasette.utils.SPATIALITE_PATHS", ["/does/not/exist"]) From 4777362bf2692bc72b221ec47c3e6216151d1b89 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 12:19:24 -0800 Subject: [PATCH 0183/1866] Work around CI bug with ensure_eventloop, refs #1115 --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 409408ae..c52960fb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,7 +67,7 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): ), ) def test_spatialite_error_if_attempt_to_open_spatialite( - spatialite_paths, should_suggest_load_extension + ensure_eventloop, spatialite_paths, should_suggest_load_extension ): with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths): runner = CliRunner() From c745c2715ab5933d7629a76bab4684632383f807 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Nov 2020 12:27:34 -0800 Subject: [PATCH 0184/1866] Moved comment for clarity --- datasette/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index ea1424a5..71c45ba0 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -247,12 +247,12 @@ class Database: ) if explicit_label_column: return explicit_label_column - # If a table has two columns, one of which is ID, then label_column is the other one column_names = await self.execute_fn(lambda conn: table_columns(conn, table)) # Is there a name or title column? name_or_title = [c for c in column_names if c in ("name", "title")] if name_or_title: return name_or_title[0] + # If a table has two columns, one of which is ID, then label_column is the other one if ( column_names and len(column_names) == 2 From 37f87b5e52e7f8ddd1c4ffcf368bd7a62a406a6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 12:01:15 -0800 Subject: [PATCH 0185/1866] Support for generated columns, closes #1116 --- datasette/utils/__init__.py | 12 +++---- tests/test_api.py | 55 ++++++++++++++++++++++++++++++-- tests/test_internals_database.py | 17 ++++++++++ 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d467383d..28df2ef1 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -64,7 +64,7 @@ HASH_LENGTH = 7 # Can replace this with Column from sqlite_utils when I add that dependency Column = namedtuple( - "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") + "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") ) @@ -460,11 +460,11 @@ def detect_primary_keys(conn, table): " Figure out primary keys for a table. " table_info_rows = [ row - for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() - if row[-1] + for row in conn.execute(f'PRAGMA table_xinfo("{table}")').fetchall() + if row["pk"] ] - table_info_rows.sort(key=lambda row: row[-1]) - return [str(r[1]) for r in table_info_rows] + table_info_rows.sort(key=lambda row: row["pk"]) + return [str(r["name"]) for r in table_info_rows] def get_outbound_foreign_keys(conn, table): @@ -572,7 +572,7 @@ def table_columns(conn, table): def table_column_details(conn, table): return [ Column(*r) - for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall() + for r in conn.execute(f"PRAGMA table_xinfo({escape_sqlite(table)});").fetchall() ] diff --git a/tests/test_api.py b/tests/test_api.py index 848daf9c..ebe50d10 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ +from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import detect_json1 +from datasette.utils import detect_json1, sqlite3 from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -514,7 +515,14 @@ def test_database_page(app_client): }, { "name": "searchable_fts", - "columns": ["text1", "text2", "name with . and spaces"], + "columns": [ + "text1", + "text2", + "name with . and spaces", + "searchable_fts", + "docid", + "__langid", + ], "primary_keys": [], "count": 2, "hidden": True, @@ -1913,3 +1921,46 @@ def test_paginate_using_link_header(app_client, qs): else: path = None assert num_pages == 21 + + +@pytest.mark.skipif( + tuple( + map( + int, + sqlite3.connect(":memory:") + .execute("select sqlite_version()") + .fetchone()[0] + .split("."), + ) + ) + < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +@pytest.mark.asyncio +async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = db_directory / "test.db" + conn = sqlite3.connect(str(db_path)) + conn.executescript( + """ + CREATE TABLE deeds ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED + ); + INSERT INTO deeds (body) VALUES ('{ + "id": 1, + "consideration": "This is the consideration" + }'); + """ + ) + datasette = Datasette([db_path]) + response = await datasette.client.get("/test/deeds.json?_shape=array") + assert response.json() == [ + { + "rowid": 1, + "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', + "id": 1, + "consideration": "This is the consideration", + } + ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 7c8f478c..56397dab 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -120,6 +120,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -128,6 +129,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=2, @@ -136,6 +138,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -144,6 +147,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -152,6 +156,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -160,6 +165,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -168,6 +174,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=7, @@ -176,6 +183,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=8, @@ -184,6 +192,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=9, @@ -192,6 +201,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), @@ -205,6 +215,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -213,6 +224,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=2, + hidden=0, ), Column( cid=2, @@ -221,6 +233,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -229,6 +242,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -237,6 +251,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -245,6 +260,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -253,6 +269,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), From dea3c508b39528e566d711c38a467b3d372d220b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 12:09:22 -0800 Subject: [PATCH 0186/1866] Revert "Support for generated columns, closes #1116" - it failed CI This reverts commit 37f87b5e52e7f8ddd1c4ffcf368bd7a62a406a6d. --- datasette/utils/__init__.py | 12 +++---- tests/test_api.py | 55 ++------------------------------ tests/test_internals_database.py | 17 ---------- 3 files changed, 8 insertions(+), 76 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 28df2ef1..d467383d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -64,7 +64,7 @@ HASH_LENGTH = 7 # Can replace this with Column from sqlite_utils when I add that dependency Column = namedtuple( - "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") + "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") ) @@ -460,11 +460,11 @@ def detect_primary_keys(conn, table): " Figure out primary keys for a table. " table_info_rows = [ row - for row in conn.execute(f'PRAGMA table_xinfo("{table}")').fetchall() - if row["pk"] + for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() + if row[-1] ] - table_info_rows.sort(key=lambda row: row["pk"]) - return [str(r["name"]) for r in table_info_rows] + table_info_rows.sort(key=lambda row: row[-1]) + return [str(r[1]) for r in table_info_rows] def get_outbound_foreign_keys(conn, table): @@ -572,7 +572,7 @@ def table_columns(conn, table): def table_column_details(conn, table): return [ Column(*r) - for r in conn.execute(f"PRAGMA table_xinfo({escape_sqlite(table)});").fetchall() + for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall() ] diff --git a/tests/test_api.py b/tests/test_api.py index ebe50d10..848daf9c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,5 @@ -from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS -from datasette.utils import detect_json1, sqlite3 +from datasette.utils import detect_json1 from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -515,14 +514,7 @@ def test_database_page(app_client): }, { "name": "searchable_fts", - "columns": [ - "text1", - "text2", - "name with . and spaces", - "searchable_fts", - "docid", - "__langid", - ], + "columns": ["text1", "text2", "name with . and spaces"], "primary_keys": [], "count": 2, "hidden": True, @@ -1921,46 +1913,3 @@ def test_paginate_using_link_header(app_client, qs): else: path = None assert num_pages == 21 - - -@pytest.mark.skipif( - tuple( - map( - int, - sqlite3.connect(":memory:") - .execute("select sqlite_version()") - .fetchone()[0] - .split("."), - ) - ) - < (3, 31, 0), - reason="generated columns were added in SQLite 3.31.0", -) -@pytest.mark.asyncio -async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = db_directory / "test.db" - conn = sqlite3.connect(str(db_path)) - conn.executescript( - """ - CREATE TABLE deeds ( - body TEXT, - id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, - consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED - ); - INSERT INTO deeds (body) VALUES ('{ - "id": 1, - "consideration": "This is the consideration" - }'); - """ - ) - datasette = Datasette([db_path]) - response = await datasette.client.get("/test/deeds.json?_shape=array") - assert response.json() == [ - { - "rowid": 1, - "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', - "id": 1, - "consideration": "This is the consideration", - } - ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 56397dab..7c8f478c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -120,7 +120,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, - hidden=0, ), Column( cid=1, @@ -129,7 +128,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=2, @@ -138,7 +136,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=3, @@ -147,7 +144,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=4, @@ -156,7 +152,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=5, @@ -165,7 +160,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=6, @@ -174,7 +168,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=7, @@ -183,7 +176,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=8, @@ -192,7 +184,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=9, @@ -201,7 +192,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), ], ), @@ -215,7 +205,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, - hidden=0, ), Column( cid=1, @@ -224,7 +213,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=2, - hidden=0, ), Column( cid=2, @@ -233,7 +221,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=3, @@ -242,7 +229,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=4, @@ -251,7 +237,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=5, @@ -260,7 +245,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), Column( cid=6, @@ -269,7 +253,6 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, - hidden=0, ), ], ), From 49b6297fb7513291110d86688c688700e6f6d9cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 13:24:23 -0800 Subject: [PATCH 0187/1866] Typo fix: messagge_is_html, closes #1118 --- datasette/app.py | 2 +- datasette/views/base.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 88d5ecc6..922046d5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1154,7 +1154,7 @@ class DatasetteRouter: status = exception.status info = exception.error_dict message = exception.message - if exception.messagge_is_html: + if exception.message_is_html: message = Markup(message) title = exception.title else: diff --git a/datasette/views/base.py b/datasette/views/base.py index bde8449f..5ba8fcb1 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -42,13 +42,13 @@ class DatasetteError(Exception): error_dict=None, status=500, template=None, - messagge_is_html=False, + message_is_html=False, ): self.message = message self.title = title self.error_dict = error_dict or {} self.status = status - self.messagge_is_html = messagge_is_html + self.message_is_html = message_is_html class BaseView: @@ -441,7 +441,7 @@ class DataView(BaseView): """, title="SQL Interrupted", status=400, - messagge_is_html=True, + message_is_html=True, ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) From 461670a0b87efa953141b449a9a261919864ceb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 13:29:57 -0800 Subject: [PATCH 0188/1866] Support for generated columns * Support for generated columns, closes #1116 * Show SQLite version in pytest report header * Use table_info() if SQLite < 3.26.0 * Cache sqlite_version() rather than re-calculate every time * Adjust test_database_page for SQLite 3.26.0 or higher --- datasette/utils/__init__.py | 41 +++++++++++++------------ datasette/utils/sqlite.py | 28 ++++++++++++++++++ tests/conftest.py | 11 +++++++ tests/fixtures.py | 2 +- tests/test_api.py | 51 +++++++++++++++++++++++++++++++- tests/test_config_dir.py | 2 +- tests/test_internals_database.py | 20 ++++++++++++- tests/test_plugins.py | 4 +-- tests/test_utils.py | 2 +- 9 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 datasette/utils/sqlite.py diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d467383d..b951539d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -19,15 +19,9 @@ import urllib import numbers import yaml from .shutil_backport import copytree +from .sqlite import sqlite3, sqlite_version from ..plugins import pm -try: - import pysqlite3 as sqlite3 -except ImportError: - import sqlite3 - -if hasattr(sqlite3, "enable_callback_tracebacks"): - sqlite3.enable_callback_tracebacks(True) # From https://www.sqlite.org/lang_keywords.html reserved_words = set( @@ -64,7 +58,7 @@ HASH_LENGTH = 7 # Can replace this with Column from sqlite_utils when I add that dependency Column = namedtuple( - "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") + "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk", "hidden") ) @@ -458,13 +452,10 @@ def temporary_docker_directory( def detect_primary_keys(conn, table): " Figure out primary keys for a table. " - table_info_rows = [ - row - for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() - if row[-1] - ] - table_info_rows.sort(key=lambda row: row[-1]) - return [str(r[1]) for r in table_info_rows] + columns = table_column_details(conn, table) + pks = [column for column in columns if column.is_pk] + pks.sort(key=lambda column: column.is_pk) + return [column.name for column in pks] def get_outbound_foreign_keys(conn, table): @@ -570,10 +561,22 @@ def table_columns(conn, table): def table_column_details(conn, table): - return [ - Column(*r) - for r in conn.execute(f"PRAGMA table_info({escape_sqlite(table)});").fetchall() - ] + if sqlite_version() >= (3, 26, 0): + # table_xinfo was added in 3.26.0 + return [ + Column(*r) + for r in conn.execute( + f"PRAGMA table_xinfo({escape_sqlite(table)});" + ).fetchall() + ] + else: + # Treat hidden as 0 for all columns + return [ + Column(*(list(r) + [0])) + for r in conn.execute( + f"PRAGMA table_info({escape_sqlite(table)});" + ).fetchall() + ] filter_column_re = re.compile(r"^_filter_column_\d+$") diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py new file mode 100644 index 00000000..9a043ccd --- /dev/null +++ b/datasette/utils/sqlite.py @@ -0,0 +1,28 @@ +try: + import pysqlite3 as sqlite3 +except ImportError: + import sqlite3 + +if hasattr(sqlite3, "enable_callback_tracebacks"): + sqlite3.enable_callback_tracebacks(True) + +_cached_sqlite_version = None + + +def sqlite_version(): + global _cached_sqlite_version + if _cached_sqlite_version is None: + _cached_sqlite_version = _sqlite_version() + return _cached_sqlite_version + + +def _sqlite_version(): + return tuple( + map( + int, + sqlite3.connect(":memory:") + .execute("select sqlite_version()") + .fetchone()[0] + .split("."), + ) + ) diff --git a/tests/conftest.py b/tests/conftest.py index 91b811e2..a963a4fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,11 @@ import pathlib import pytest import re +try: + import pysqlite3 as sqlite3 +except ImportError: + import sqlite3 + UNDOCUMENTED_PERMISSIONS = { "this_is_allowed", "this_is_denied", @@ -12,6 +17,12 @@ UNDOCUMENTED_PERMISSIONS = { } +def pytest_report_header(config): + return "SQLite: {}".format( + sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] + ) + + def pytest_configure(config): import sys diff --git a/tests/fixtures.py b/tests/fixtures.py index f95a2d6b..b0c98f39 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils import sqlite3 +from datasette.utils.sqlite import sqlite3 from datasette.utils.testing import TestClient import click import contextlib diff --git a/tests/test_api.py b/tests/test_api.py index 848daf9c..5676622e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,7 @@ +from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 +from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -514,7 +516,20 @@ def test_database_page(app_client): }, { "name": "searchable_fts", - "columns": ["text1", "text2", "name with . and spaces"], + "columns": [ + "text1", + "text2", + "name with . and spaces", + ] + + ( + [ + "searchable_fts", + "docid", + "__langid", + ] + if sqlite_version() >= (3, 26, 0) + else [] + ), "primary_keys": [], "count": 2, "hidden": True, @@ -1913,3 +1928,37 @@ def test_paginate_using_link_header(app_client, qs): else: path = None assert num_pages == 21 + + +@pytest.mark.skipif( + sqlite_version() < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +@pytest.mark.asyncio +async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = db_directory / "test.db" + conn = sqlite3.connect(str(db_path)) + conn.executescript( + """ + CREATE TABLE deeds ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED + ); + INSERT INTO deeds (body) VALUES ('{ + "id": 1, + "consideration": "This is the consideration" + }'); + """ + ) + datasette = Datasette([db_path]) + response = await datasette.client.get("/test/deeds.json?_shape=array") + assert response.json() == [ + { + "rowid": 1, + "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', + "id": 1, + "consideration": "This is the consideration", + } + ] diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index cd158474..015c6ace 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,9 +1,9 @@ import json import pytest -import sqlite3 from datasette.app import Datasette from datasette.cli import cli +from datasette.utils.sqlite import sqlite3 from .fixtures import TestClient as _TestClient from click.testing import CliRunner diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 7c8f478c..e50cf20e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,7 +2,8 @@ Tests for the datasette.database.Database class """ from datasette.database import Database, Results, MultipleValues -from datasette.utils import sqlite3, Column +from datasette.utils.sqlite import sqlite3 +from datasette.utils import Column from .fixtures import app_client import pytest import time @@ -120,6 +121,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -128,6 +130,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=2, @@ -136,6 +139,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -144,6 +148,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -152,6 +157,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -160,6 +166,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -168,6 +175,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=7, @@ -176,6 +184,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=8, @@ -184,6 +193,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=9, @@ -192,6 +202,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), @@ -205,6 +216,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=1, + hidden=0, ), Column( cid=1, @@ -213,6 +225,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=2, + hidden=0, ), Column( cid=2, @@ -221,6 +234,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=3, @@ -229,6 +243,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=4, @@ -237,6 +252,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=5, @@ -245,6 +261,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), Column( cid=6, @@ -253,6 +270,7 @@ async def test_table_columns(db, table, expected): notnull=0, default_value=None, is_pk=0, + hidden=0, ), ], ), diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 51faeccb..4554cfd4 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,14 +9,14 @@ from .fixtures import ( from datasette.app import Datasette from datasette import cli from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm -from datasette.utils import sqlite3, CustomRow +from datasette.utils.sqlite import sqlite3 +from datasette.utils import CustomRow from jinja2.environment import Template import base64 import json import os import pathlib import re -import sqlite3 import textwrap import pytest import urllib diff --git a/tests/test_utils.py b/tests/test_utils.py index 07e6f870..56306339 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,11 +4,11 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request +from datasette.utils.sqlite import sqlite3 import json import os import pathlib import pytest -import sqlite3 import tempfile from unittest.mock import patch From 17cbbb1f7f230b39650afac62dd16476626001b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 16:28:02 -0800 Subject: [PATCH 0189/1866] generated_columns table in fixtures.py, closes #1119 --- datasette/utils/__init__.py | 4 +- datasette/utils/sqlite.py | 8 ++++ tests/fixtures.py | 19 ++++++++- tests/test_api.py | 51 ++++++++++++----------- tests/test_internals_database.py | 70 +++++++++++++++++--------------- tests/test_plugins.py | 6 +-- 6 files changed, 93 insertions(+), 65 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index b951539d..2576090a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -19,7 +19,7 @@ import urllib import numbers import yaml from .shutil_backport import copytree -from .sqlite import sqlite3, sqlite_version +from .sqlite import sqlite3, sqlite_version, supports_table_xinfo from ..plugins import pm @@ -561,7 +561,7 @@ def table_columns(conn, table): def table_column_details(conn, table): - if sqlite_version() >= (3, 26, 0): + if supports_table_xinfo(): # table_xinfo was added in 3.26.0 return [ Column(*r) diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 9a043ccd..c8522f35 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -26,3 +26,11 @@ def _sqlite_version(): .split("."), ) ) + + +def supports_table_xinfo(): + return sqlite_version() >= (3, 26, 0) + + +def supports_generated_columns(): + return sqlite_version() >= (3, 31, 0) diff --git a/tests/fixtures.py b/tests/fixtures.py index b0c98f39..b52a531f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, sqlite_version, supports_generated_columns from datasette.utils.testing import TestClient import click import contextlib @@ -116,6 +116,8 @@ def make_app_client( immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) + if supports_generated_columns(): + conn.executescript(GENERATED_COLUMNS_SQL) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) @@ -699,6 +701,18 @@ INSERT INTO "searchable_fts" (rowid, text1, text2) SELECT rowid, text1, text2 FROM searchable; """ +GENERATED_COLUMNS_SQL = """ +CREATE TABLE generated_columns ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED +); +INSERT INTO generated_columns (body) VALUES ('{ + "number": 1, + "string": "This is a string" +}'); +""" + def assert_permissions_checked(datasette, actions): # actions is a list of "action" or (action, resource) tuples @@ -754,6 +768,9 @@ def cli(db_filename, metadata, plugins_path, recreate): for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) + if supports_generated_columns(): + with conn: + conn.executescript(GENERATED_COLUMNS_SQL) print(f"Test tables written to {db_filename}") if metadata: open(metadata, "w").write(json.dumps(METADATA, indent=4)) diff --git a/tests/test_api.py b/tests/test_api.py index 5676622e..f82a8fe9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,7 +1,7 @@ from datasette.app import Datasette from datasette.plugins import DEFAULT_PLUGINS from datasette.utils import detect_json1 -from datasette.utils.sqlite import sqlite3, sqlite_version +from datasette.utils.sqlite import sqlite3, sqlite_version, supports_table_xinfo from datasette.version import __version__ from .fixtures import ( # noqa app_client, @@ -19,6 +19,7 @@ from .fixtures import ( # noqa generate_compound_rows, generate_sortable_rows, make_app_client, + supports_generated_columns, EXPECTED_PLUGINS, METADATA, ) @@ -35,7 +36,7 @@ def test_homepage(app_client): assert response.json.keys() == {"fixtures": 0}.keys() d = response.json["fixtures"] assert d["name"] == "fixtures" - assert d["tables_count"] == 24 + assert d["tables_count"] == 25 if supports_generated_columns() else 24 assert len(d["tables_and_views_truncated"]) == 5 assert d["tables_and_views_more"] is True # 4 hidden FTS tables + no_primary_key (hidden in metadata) @@ -268,6 +269,22 @@ def test_database_page(app_client): }, "private": False, }, + ] + ( + [ + { + "columns": ["body", "id", "consideration"], + "count": 1, + "foreign_keys": {"incoming": [], "outgoing": []}, + "fts_table": None, + "hidden": False, + "name": "generated_columns", + "primary_keys": [], + "private": False, + } + ] + if supports_generated_columns() + else [] + ) + [ { "name": "infinity", "columns": ["value"], @@ -527,7 +544,7 @@ def test_database_page(app_client): "docid", "__langid", ] - if sqlite_version() >= (3, 26, 0) + if supports_table_xinfo() else [] ), "primary_keys": [], @@ -1934,31 +1951,13 @@ def test_paginate_using_link_header(app_client, qs): sqlite_version() < (3, 31, 0), reason="generated columns were added in SQLite 3.31.0", ) -@pytest.mark.asyncio -async def test_generated_columns_are_visible_in_datasette(tmp_path_factory): - db_directory = tmp_path_factory.mktemp("dbs") - db_path = db_directory / "test.db" - conn = sqlite3.connect(str(db_path)) - conn.executescript( - """ - CREATE TABLE deeds ( - body TEXT, - id INT GENERATED ALWAYS AS (json_extract(body, '$.id')) STORED, - consideration INT GENERATED ALWAYS AS (json_extract(body, '$.consideration')) STORED - ); - INSERT INTO deeds (body) VALUES ('{ - "id": 1, - "consideration": "This is the consideration" - }'); - """ - ) - datasette = Datasette([db_path]) - response = await datasette.client.get("/test/deeds.json?_shape=array") +async def test_generated_columns_are_visible_in_datasette(app_client): + response = app_client.get("/test/generated_columns.json?_shape=array") assert response.json() == [ { "rowid": 1, - "body": '{\n "id": 1,\n "consideration": "This is the consideration"\n }', - "id": 1, - "consideration": "This is the consideration", + "body": '{\n "number": 1,\n "string": "This is a string"\n }', + "number": 1, + "string": "This is a string", } ] diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e50cf20e..49b8a1b3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,7 +2,7 @@ Tests for the datasette.database.Database class """ from datasette.database import Database, Results, MultipleValues -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, supports_generated_columns from datasette.utils import Column from .fixtures import app_client import pytest @@ -340,38 +340,42 @@ async def test_get_all_foreign_keys(db): @pytest.mark.asyncio async def test_table_names(db): table_names = await db.table_names() - assert table_names == [ - "simple_primary_key", - "primary_key_multiple_columns", - "primary_key_multiple_columns_explicit_label", - "compound_primary_key", - "compound_three_primary_keys", - "foreign_key_references", - "sortable", - "no_primary_key", - "123_starts_with_digits", - "Table With Space In Name", - "table/with/slashes.csv", - "complex_foreign_keys", - "custom_foreign_key_label", - "units", - "tags", - "searchable", - "searchable_tags", - "searchable_fts", - "searchable_fts_segments", - "searchable_fts_segdir", - "searchable_fts_docsize", - "searchable_fts_stat", - "select", - "infinity", - "facet_cities", - "facetable", - "binary_data", - "roadside_attractions", - "attraction_characteristic", - "roadside_attraction_characteristics", - ] + assert ( + table_names + == [ + "simple_primary_key", + "primary_key_multiple_columns", + "primary_key_multiple_columns_explicit_label", + "compound_primary_key", + "compound_three_primary_keys", + "foreign_key_references", + "sortable", + "no_primary_key", + "123_starts_with_digits", + "Table With Space In Name", + "table/with/slashes.csv", + "complex_foreign_keys", + "custom_foreign_key_label", + "units", + "tags", + "searchable", + "searchable_tags", + "searchable_fts", + "searchable_fts_segments", + "searchable_fts_segdir", + "searchable_fts_docsize", + "searchable_fts_stat", + "select", + "infinity", + "facet_cities", + "facetable", + "binary_data", + "roadside_attractions", + "attraction_characteristic", + "roadside_attraction_characteristics", + ] + + (["generated_columns"] if supports_generated_columns() else []) + ) @pytest.mark.asyncio diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4554cfd4..dab5ef68 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -413,8 +413,7 @@ def test_hook_register_output_renderer_all_parameters(app_client): # Lots of 'at 0x103a4a690' in here - replace those so we can do # an easy comparison body = at_memory_re.sub(" at 0xXXX", response.text) - assert { - "1+1": 2, + assert json.loads(body) == { "datasette": "", "columns": [ "pk", @@ -451,7 +450,8 @@ def test_hook_register_output_renderer_all_parameters(app_client): "table": "facetable", "request": "", "view_name": "table", - } == json.loads(body) + "1+1": 2, + } # Test that query_name is set correctly query_response = app_client.get("/fixtures/pragma_cache_size.testall") assert "pragma_cache_size" == json.loads(query_response.body)["query_name"] From a970276b9999687b96c5e11ea1c817d814f5d267 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Nov 2020 17:19:09 -0800 Subject: [PATCH 0190/1866] Try pysqlite3 on latest.datasette.io --install=pysqlite3-binary to get a working demo of generated columns, refs #1119 --- .github/workflows/deploy-latest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7a41bda2..05f0bad1 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -54,6 +54,7 @@ jobs: --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ --extra-options="--setting template_debug 1" \ + --install=pysqlite3-binary \ --service=datasette-latest # Deploy docs.db to a different service datasette publish cloudrun docs.db \ From 88ac538b41a4753c3de9b509c3a0e13077f66182 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Dec 2020 15:47:37 -0800 Subject: [PATCH 0191/1866] transfer-encoding: chunked for DB downloads, refs #749 This should get >32MB downloads working on Cloud Run. --- datasette/views/database.py | 1 + tests/test_html.py | 1 + 2 files changed, 2 insertions(+) diff --git a/datasette/views/database.py b/datasette/views/database.py index 17c78150..f6fd579c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -169,6 +169,7 @@ class DatabaseDownload(DataView): headers = {} if self.ds.cors: headers["Access-Control-Allow-Origin"] = "*" + headers["Transfer-Encoding"] = "chunked" return AsgiFileDownload( filepath, filename=os.path.basename(filepath), diff --git a/tests/test_html.py b/tests/test_html.py index ecbf89b4..b9d3afcd 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1028,6 +1028,7 @@ def test_database_download_for_immutable(): download_response.headers["content-disposition"] == 'attachment; filename="fixtures.db"' ) + assert download_response.headers["transfer-encoding"] == "chunked" def test_database_download_disallowed_for_mutable(app_client): From daae35be46ec5cb8a207aa20986a4fa62e94777e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abdussamet=20Ko=C3=A7ak?= Date: Thu, 3 Dec 2020 03:33:36 +0300 Subject: [PATCH 0192/1866] Fix misaligned table actions cog Closes #1121. Thanks, @abdusco --- datasette/static/app.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index b9378a9e..9e498ab9 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -163,6 +163,8 @@ h6, } .page-header { + display: flex; + align-items: center; padding-left: 10px; border-left: 10px solid #666; margin-bottom: 0.75rem; @@ -175,11 +177,11 @@ h6, padding-right: 0.2em; } .page-header details { - display: inline; + display: inline-flex; } .page-header details > summary { list-style: none; - display: inline; + display: inline-flex; cursor: pointer; } .page-header details > summary::-webkit-details-marker { From a45a3dff3ea01a2382dcedae5923a7b821a12aec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Dec 2020 16:44:03 -0800 Subject: [PATCH 0193/1866] Fix for OPTIONS request against /db, closes #1100 --- datasette/utils/testing.py | 23 +++++++++++++++++++++++ datasette/views/base.py | 2 +- tests/test_api.py | 6 ++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index bcbc1c7a..57b19ea5 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -99,6 +99,29 @@ class TestClient: content_type=content_type, ) + @async_to_sync + async def request( + self, + path, + allow_redirects=True, + redirect_count=0, + method="GET", + cookies=None, + headers=None, + post_body=None, + content_type=None, + ): + return await self._request( + path, + allow_redirects=allow_redirects, + redirect_count=redirect_count, + method=method, + cookies=cookies, + headers=headers, + post_body=post_body, + content_type=content_type, + ) + async def _request( self, path, diff --git a/datasette/views/base.py b/datasette/views/base.py index 5ba8fcb1..a93a6378 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -155,7 +155,7 @@ class DataView(BaseView): name = "" re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") - def options(self, request, *args, **kwargs): + async def options(self, request, *args, **kwargs): r = Response.text("ok") if self.ds.cors: r.headers["Access-Control-Allow-Origin"] = "*" diff --git a/tests/test_api.py b/tests/test_api.py index f82a8fe9..016894b4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1961,3 +1961,9 @@ async def test_generated_columns_are_visible_in_datasette(app_client): "string": "This is a string", } ] + + +def test_http_options_request(app_client): + response = app_client.request("/fixtures", method="OPTIONS") + assert response.status == 200 + assert response.text == "ok" From 13c960c03b46e35f3432063a19f3f528ca249e23 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Dec 2020 16:49:43 -0800 Subject: [PATCH 0194/1866] Test is no longer order dependent, closes #1123 --- tests/test_plugins.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index dab5ef68..93b444ab 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -797,9 +797,11 @@ def test_hook_table_actions(app_client, table_or_view): assert get_table_actions_links(response.text) == [] response_2 = app_client.get(f"/fixtures/{table_or_view}?_bot=1") - assert get_table_actions_links(response_2.text) == [ - {"label": "From async", "href": "/"}, + assert sorted( + get_table_actions_links(response_2.text), key=lambda l: l["label"] + ) == [ {"label": "Database: fixtures", "href": "/"}, + {"label": "From async", "href": "/"}, {"label": f"Table: {table_or_view}", "href": "/"}, ] From e048791a9a2686f47d81a2c8aa88aa1966d82521 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Dec 2020 16:57:40 -0800 Subject: [PATCH 0195/1866] Release 0.52.2 Refs #1116, #1115, #1100, #749, #1121 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 119295b3..0353358a 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52.1" +__version__ = "0.52.2" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index a77cf5a5..6fb06beb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_52_2: + +0.52.2 (2020-12-02) +------------------- + +- Generated columns from SQLite 3.31.0 or higher are now correctly displayed. (`#1116 `__) +- Error message if you attempt to open a SpatiaLite database now suggests using ``--load-extension=spatialite`` if it detects that the extension is available in a common location. (`#1115 `__) +- ``OPTIONS`` requests against the ``/database`` page no longer raise a 500 error. (`#1100 `__) +- Databases larger than 32MB that are published to Cloud Run can now be downloaded. (`#749 `__) +- Fix for misaligned cog icon on table and database pages. Thanks, Abdussamet Koçak. (`#1121 `__) + .. _v0_52_1: 0.52.1 (2020-11-29) From 6b4c55efea3e9d34d92cbe5f0066553ad9b14071 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 10:53:26 -0800 Subject: [PATCH 0196/1866] Fix for Amazon Linux static assets 404ing, refs #1124 --- datasette/utils/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index ce78a597..31b0bdcd 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -294,7 +294,7 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): return # Ensure full_path is within root_path to avoid weird "../" tricks try: - full_path.relative_to(root_path) + full_path.relative_to(root_path.resolve()) except ValueError: await asgi_send_html(send, "404", 404) return From 63efcb35ce879fe68ee02411c8dd2fd5f127cc32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 11:02:53 -0800 Subject: [PATCH 0197/1866] More tweaks to root_path handling, refs #1124 --- datasette/utils/asgi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 31b0bdcd..3b41c2d7 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -282,10 +282,12 @@ async def asgi_send_file( def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): + root_path = Path(root_path) + async def inner_static(request, send): path = request.scope["url_route"]["kwargs"]["path"] try: - full_path = (Path(root_path) / path).resolve().absolute() + full_path = (root_path / path).resolve().absolute() except FileNotFoundError: await asgi_send_html(send, "404", 404) return From ca6e8e53dc9b094a5ce169d81a69d872546e595a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 11:05:12 -0800 Subject: [PATCH 0198/1866] More helpful 404 messages, refs #1124 --- datasette/utils/asgi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 3b41c2d7..363f059f 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -289,7 +289,7 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): try: full_path = (root_path / path).resolve().absolute() except FileNotFoundError: - await asgi_send_html(send, "404", 404) + await asgi_send_html(send, "404: Directory not found", 404) return if full_path.is_dir(): await asgi_send_html(send, "403: Directory listing is not allowed", 403) @@ -298,12 +298,12 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): try: full_path.relative_to(root_path.resolve()) except ValueError: - await asgi_send_html(send, "404", 404) + await asgi_send_html(send, "404: Path not inside root path", 404) return try: await asgi_send_file(send, full_path, chunk_size=chunk_size) except FileNotFoundError: - await asgi_send_html(send, "404", 404) + await asgi_send_html(send, "404: File not found", 404) return return inner_static From 4cce5516661b24afeddaf35bee84b00fbf5c7f89 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 11:07:05 -0800 Subject: [PATCH 0199/1866] Release 0.52.3 Refs #1124 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0353358a..ab02947d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52.2" +__version__ = "0.52.3" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6fb06beb..4fa7609c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_52_3: + +0.52.3 (2020-12-03) +------------------- + +- Fixed bug where static assets would 404 for Datasette installed on ARM Amazon Linux. (`#1124 `__) + .. _v0_52_2: 0.52.2 (2020-12-02) From 00185af74a91646d47aa54f2369c1a19a6f76a27 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 14:08:50 -0800 Subject: [PATCH 0200/1866] Show pysqlite3 version on /-/versions, if installed - #1125 --- datasette/app.py | 14 ++++++++++++-- datasette/utils/sqlite.py | 3 +++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 922046d5..b2f16257 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -9,6 +9,7 @@ import inspect from itsdangerous import BadSignature import json import os +import pkg_resources import re import secrets import sys @@ -57,7 +58,6 @@ from .utils import ( module_from_path, parse_metadata, resolve_env_secrets, - sqlite3, to_css_class, HASH_LENGTH, ) @@ -74,6 +74,10 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) +from .utils.sqlite import ( + sqlite3, + using_pysqlite3, +) from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ @@ -619,7 +623,7 @@ class Datasette: datasette_version = {"version": __version__} if self.version_note: datasette_version["note"] = self.version_note - return { + info = { "python": { "version": ".".join(map(str, sys.version_info[:3])), "full": sys.version, @@ -636,6 +640,12 @@ class Datasette: ], }, } + if using_pysqlite3: + try: + info["pysqlite3"] = pkg_resources.get_distribution("pysqlite3").version + except pkg_resources.DistributionNotFound: + pass + return info def _plugins(self, request=None, all=False): ps = list(get_plugins()) diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index c8522f35..342ff3fa 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -1,5 +1,8 @@ +using_pysqlite3 = False try: import pysqlite3 as sqlite3 + + using_pysqlite3 = True except ImportError: import sqlite3 From e2fea36540e952d8d72c1bd0af7144b85b7a4671 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 19:12:33 -0800 Subject: [PATCH 0201/1866] Switch to google-github-actions/setup-gcloud - refs #1126 --- .github/workflows/deploy-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 05f0bad1..2de0a8b6 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -39,7 +39,7 @@ jobs: sphinx-to-sqlite ../docs.db _build cd .. - name: Set up Cloud Run - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + uses: google-github-actions/setup-gcloud@master with: version: '275.0.0' service_account_email: ${{ secrets.GCP_SA_EMAIL }} From 49d8fc056844d5a537d6cfd96dab0dd5686fe718 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 3 Dec 2020 20:07:10 -0800 Subject: [PATCH 0202/1866] Try pysqlite3-binary version as well, refs #1125 --- datasette/app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b2f16257..9bc84df0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -641,10 +641,12 @@ class Datasette: }, } if using_pysqlite3: - try: - info["pysqlite3"] = pkg_resources.get_distribution("pysqlite3").version - except pkg_resources.DistributionNotFound: - pass + for package in ("pysqlite3", "pysqlite3-binary"): + try: + info["pysqlite3"] = pkg_resources.get_distribution(package).version + break + except pkg_resources.DistributionNotFound: + pass return info def _plugins(self, request=None, all=False): From 42efb799ea9b362f0c7598f3ff3c4bf46c18e53f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 4 Dec 2020 21:20:12 -0800 Subject: [PATCH 0203/1866] Fixed invalid test for generated columns, refs #1119 --- tests/test_api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 016894b4..4339507c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1951,14 +1951,14 @@ def test_paginate_using_link_header(app_client, qs): sqlite_version() < (3, 31, 0), reason="generated columns were added in SQLite 3.31.0", ) -async def test_generated_columns_are_visible_in_datasette(app_client): - response = app_client.get("/test/generated_columns.json?_shape=array") - assert response.json() == [ +def test_generated_columns_are_visible_in_datasette(app_client): + response = app_client.get("/fixtures/generated_columns.json?_shape=array") + assert response.json == [ { "rowid": 1, - "body": '{\n "number": 1,\n "string": "This is a string"\n }', - "number": 1, - "string": "This is a string", + "body": '{\n "number": 1,\n "string": "This is a string"\n}', + "id": 1, + "consideration": "This is a string", } ] From eae103a82b92949189cf718794d2ad0424005460 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 4 Dec 2020 21:21:11 -0800 Subject: [PATCH 0204/1866] Write errors to stderr, closes #1131 --- datasette/database.py | 10 ++++++---- datasette/renderer.py | 1 - datasette/views/base.py | 4 +++- tests/test_cli.py | 7 +++++++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 71c45ba0..412e0c59 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,8 +1,8 @@ import asyncio -import contextlib from pathlib import Path import janus import queue +import sys import threading import uuid @@ -104,7 +104,8 @@ class Database: try: result = task.fn(conn) except Exception as e: - print(e) + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() result = e task.reply_queue.sync_q.put(result) @@ -156,11 +157,12 @@ class Database: if e.args == ("interrupted",): raise QueryInterrupted(e, sql, params) if log_sql_errors: - print( - "ERROR: conn={}, sql = {}, params = {}: {}".format( + sys.stderr.write( + "ERROR: conn={}, sql = {}, params = {}: {}\n".format( conn, repr(sql), params, e ) ) + sys.stderr.flush() raise if truncate: diff --git a/datasette/renderer.py b/datasette/renderer.py index d779b44f..258199fc 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -20,7 +20,6 @@ def convert_specific_columns_to_json(rows, columns, json_cols): try: value = json.loads(value) except (TypeError, ValueError) as e: - print(e) pass new_row.append(value) new_rows.append(new_row) diff --git a/datasette/views/base.py b/datasette/views/base.py index a93a6378..b8860b74 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -2,6 +2,7 @@ import asyncio import csv import hashlib import re +import sys import time import urllib @@ -362,7 +363,8 @@ class DataView(BaseView): new_row.append(cell) await writer.writerow(new_row) except Exception as e: - print("caught this", e) + sys.stderr.write("Caught this error: {}\n".format(e)) + sys.stderr.flush() await r.write(str(e)) return diff --git a/tests/test_cli.py b/tests/test_cli.py index c52960fb..a0ac7d7a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -214,3 +214,10 @@ def test_config_deprecated(ensure_eventloop): assert result.exit_code == 0 assert not json.loads(result.output)["allow_download"] assert "will be deprecated in" in result.stderr + + +def test_sql_errors_logged_to_stderr(ensure_eventloop): + runner = CliRunner(mix_stderr=False) + result = runner.invoke(cli, ["--get", "/:memory:.json?sql=select+blah"]) + assert result.exit_code == 1 + assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr From 705d1a1555c4791e9be3b884285b047223ab184f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abdussamet=20Ko=C3=A7ak?= Date: Sat, 5 Dec 2020 22:35:03 +0300 Subject: [PATCH 0205/1866] Fix startup error on windows (#1128) Fixes https://github.com/simonw/datasette/issues/1094 This import isn't used at all, and causes error on startup on Windows. --- datasette/utils/asgi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 363f059f..fc9adcff 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,5 +1,4 @@ import json -from os import EX_CANTCREAT from datasette.utils import MultiParams from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl From 2dc281645a76c550789ede80c1bc6f733fa9a82e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Dec 2020 11:41:40 -0800 Subject: [PATCH 0206/1866] Release 0.52.4 Refs #1125, #1131, #1094 --- datasette/version.py | 2 +- docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index ab02947d..ce06fe1d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52.3" +__version__ = "0.52.4" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fa7609c..a9922ab3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_52_4: + +0.52.4 (2020-12-05) +------------------- + +- Show `pysqlite3 `__ version on ``/-/versions``, if installed. (`#1125 `__) +- Errors output by Datasette (e.g. for invalid SQL queries) now go to ``stderr``, not ``stdout``. (`#1131 `__) +- Fix for a startup error on windows caused by unneccessary ``from os import EX_CANTCREAT`` - thanks, Abdussamet Koçak. (`#1094 `__) + .. _v0_52_3: 0.52.3 (2020-12-03) From e5930e6f889617320454ab53ecc1c438377d49e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Dec 2020 11:42:42 -0800 Subject: [PATCH 0207/1866] Typo fix in release notes --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a9922ab3..86d844f7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,7 @@ Changelog - Show `pysqlite3 `__ version on ``/-/versions``, if installed. (`#1125 `__) - Errors output by Datasette (e.g. for invalid SQL queries) now go to ``stderr``, not ``stdout``. (`#1131 `__) -- Fix for a startup error on windows caused by unneccessary ``from os import EX_CANTCREAT`` - thanks, Abdussamet Koçak. (`#1094 `__) +- Fix for a startup error on windows caused by unnecessary ``from os import EX_CANTCREAT`` - thanks, Abdussamet Koçak. (`#1094 `__) .. _v0_52_3: From e3143700a245d87bc532d44867b2e380b4225324 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Dec 2020 11:00:10 -0800 Subject: [PATCH 0208/1866] Custom template for docs, linking to datasette.io --- docs/_templates/layout.html | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 00000000..b7b6f794 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{%- extends "!layout.html" %} + +{% block sidebartitle %} + + + + + +{% if theme_display_version %} + {%- set nav_version = version %} + {% if READTHEDOCS and current_version %} + {%- set nav_version = current_version %} + {% endif %} + {% if nav_version %} +
    + {{ nav_version }} +
    + {% endif %} +{% endif %} + +{% include "searchbox.html" %} + +{% endblock %} From 62a6f70c64e4d04c15d9f386dcdf9cd465bbb0f6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Dec 2020 12:10:05 -0800 Subject: [PATCH 0209/1866] Fixed Markdown indentation of news To make it easier to programmatically extract. --- README.md | 76 +++++++++++++++++++++++++++---------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index c0019e9b..89245cf1 100644 --- a/README.md +++ b/README.md @@ -25,53 +25,53 @@ Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly new ## News - * 28th November 2020: [Datasette 0.52](https://docs.datasette.io/en/stable/changelog.html#v0-52) - `--config` is now `--setting`, new `database_actions` plugin hook, `datasette publish cloudrun --apt-get-install` option and several bug fixes. - * 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. [Annotated release notes](https://simonwillison.net/2020/Nov/1/datasette-0-51/). - * 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). - * 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). - * 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. - * 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. - * 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more. - * 6th August 2020: [GraphQL in Datasette with the new datasette-graphql plugin](https://simonwillison.net/2020/Aug/7/datasette-graphql/) - * 24th July 2020: Two new plugins: [datasette-copyable and datasette-insert-api](https://simonwillison.net/2020/Jul/23/datasette-copyable-datasette-insert-api/). `datasette-copyable` adds copy-and-paste export options, and `datasette-insert-api` lets you create tables and insert or update data by POSTing JSON directly to Datasette. - * 1st July 2020: [Datasette 0.45](https://docs.datasette.io/en/stable/changelog.html#v0-45) - [Magic parameters for canned queries](https://docs.datasette.io/en/stable/sql_queries.html#canned-queries-magic-parameters), a log out feature, improved plugin documentation and four new plugin hooks. See also [Datasette 0.45: The annotated release notes](https://simonwillison.net/2020/Jul/1/datasette-045/). - * 20th June 2020: [A cookiecutter template for writing Datasette plugins](https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/) - * 11th June 2020: [Datasette 0.44](https://docs.datasette.io/en/stable/changelog.html#v0-44) - [Authentication and permissions](https://docs.datasette.io/en/stable/authentication.html), [writable canned queries](https://docs.datasette.io/en/stable/sql_queries.html#writable-canned-queries), flash messages, new plugin hooks and much, much more. - * 28th May 2020: [Datasette 0.43](https://docs.datasette.io/en/stable/changelog.html#v0-43) - Redesigned [register_output_renderer](https://docs.datasette.io/en/stable/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes. - * 8th May 2020: [Datasette 0.42](https://docs.datasette.io/en/stable/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database. - * 6th May 2020: [Datasette 0.41](https://docs.datasette.io/en/stable/changelog.html#v0-41) - New mechanism for [creating custom pages](https://docs.datasette.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://docs.datasette.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. - * 21st April 2020: [Datasette 0.40](https://docs.datasette.io/en/stable/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. - * 24th March 2020: [Datasette 0.39](https://docs.datasette.io/en/stable/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table. - * 8th March 2020: [Datasette 0.38](https://docs.datasette.io/en/stable/changelog.html#v0-38) - New `--memory` option for `datasete publish cloudrun`, [Docker image](https://hub.docker.com/r/datasetteproject/datasette) upgraded to SQLite 3.31.1. - * 25th February 2020: [Datasette 0.37](https://docs.datasette.io/en/stable/changelog.html#v0-37) - new internal APIs enabling plugins to safely write to databases. Read more here: [Datasette Writes](https://simonwillison.net/2020/Feb/26/weeknotes-datasette-writes/). - * 21st February 2020: [Datasette 0.36](https://docs.datasette.io/en/stable/changelog.html#v0-36) - new internals documentation for plugins, `prepare_connection()` now accepts optional `database` and `datasette` arguments. - * 4th February 2020: [Datasette 0.35](https://docs.datasette.io/en/stable/changelog.html#v0-35) - new `.render_template()` method for plugins. - * 29th January 2020: [Datasette 0.34](https://docs.datasette.io/en/stable/changelog.html#v0-34) - improvements to search, `datasette publish cloudrun` and `datasette package`. - * 21st January 2020: [Deploying a data API using GitHub Actions and Cloud Run](https://simonwillison.net/2020/Jan/21/github-actions-cloud-run/) - how to use GitHub Actions and Google Cloud Run to automatically scrape data and deploy the result as an API with Datasette. - * 22nd December 2019: [Datasette 0.33](https://docs.datasette.io/en/stable/changelog.html#v0-33) - various small improvements. - * 19th December 2019: [Building tools to bring data-driven reporting to more newsrooms](https://medium.com/jsk-class-of-2020/building-tools-to-bring-data-driven-reporting-to-more-newsrooms-4520a0c9b3f2) - some notes on my JSK fellowship so far. - * 2nd December 2019: [Niche Museums](https://www.niche-museums.com/) is a new site entirely powered by Datasette, using custom templates and plugins. [niche-museums.com, powered by Datasette](https://simonwillison.net/2019/Nov/25/niche-museums/) describes how the site works, and [datasette-atom: Define an Atom feed using a custom SQL query](https://simonwillison.net/2019/Dec/3/datasette-atom/) describes how the new [datasette-atom plugin](https://github.com/simonw/datasette-atom) was used to add an Atom syndication feed to the site. - * 14th November 2019: [Datasette 0.32](https://docs.datasette.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function. - * 11th November 2019: [Datasette 0.31](https://docs.datasette.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. - * 18th October 2019: [Datasette 0.30](https://docs.datasette.io/en/stable/changelog.html#v0-30) - * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. - * 7th July 2019: [Datasette 0.29](https://docs.datasette.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... +* 28th November 2020: [Datasette 0.52](https://docs.datasette.io/en/stable/changelog.html#v0-52) - `--config` is now `--setting`, new `database_actions` plugin hook, `datasette publish cloudrun --apt-get-install` option and several bug fixes. +* 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. [Annotated release notes](https://simonwillison.net/2020/Nov/1/datasette-0-51/). +* 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). +* 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). +* 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. +* 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. +* 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more. +* 6th August 2020: [GraphQL in Datasette with the new datasette-graphql plugin](https://simonwillison.net/2020/Aug/7/datasette-graphql/) +* 24th July 2020: Two new plugins: [datasette-copyable and datasette-insert-api](https://simonwillison.net/2020/Jul/23/datasette-copyable-datasette-insert-api/). `datasette-copyable` adds copy-and-paste export options, and `datasette-insert-api` lets you create tables and insert or update data by POSTing JSON directly to Datasette. +* 1st July 2020: [Datasette 0.45](https://docs.datasette.io/en/stable/changelog.html#v0-45) - [Magic parameters for canned queries](https://docs.datasette.io/en/stable/sql_queries.html#canned-queries-magic-parameters), a log out feature, improved plugin documentation and four new plugin hooks. See also [Datasette 0.45: The annotated release notes](https://simonwillison.net/2020/Jul/1/datasette-045/). +* 20th June 2020: [A cookiecutter template for writing Datasette plugins](https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/) +* 11th June 2020: [Datasette 0.44](https://docs.datasette.io/en/stable/changelog.html#v0-44) - [Authentication and permissions](https://docs.datasette.io/en/stable/authentication.html), [writable canned queries](https://docs.datasette.io/en/stable/sql_queries.html#writable-canned-queries), flash messages, new plugin hooks and much, much more. +* 28th May 2020: [Datasette 0.43](https://docs.datasette.io/en/stable/changelog.html#v0-43) - Redesigned [register_output_renderer](https://docs.datasette.io/en/stable/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes. +* 8th May 2020: [Datasette 0.42](https://docs.datasette.io/en/stable/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database. +* 6th May 2020: [Datasette 0.41](https://docs.datasette.io/en/stable/changelog.html#v0-41) - New mechanism for [creating custom pages](https://docs.datasette.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://docs.datasette.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. +* 21st April 2020: [Datasette 0.40](https://docs.datasette.io/en/stable/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. +* 24th March 2020: [Datasette 0.39](https://docs.datasette.io/en/stable/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table. +* 8th March 2020: [Datasette 0.38](https://docs.datasette.io/en/stable/changelog.html#v0-38) - New `--memory` option for `datasete publish cloudrun`, [Docker image](https://hub.docker.com/r/datasetteproject/datasette) upgraded to SQLite 3.31.1. +* 25th February 2020: [Datasette 0.37](https://docs.datasette.io/en/stable/changelog.html#v0-37) - new internal APIs enabling plugins to safely write to databases. Read more here: [Datasette Writes](https://simonwillison.net/2020/Feb/26/weeknotes-datasette-writes/). +* 21st February 2020: [Datasette 0.36](https://docs.datasette.io/en/stable/changelog.html#v0-36) - new internals documentation for plugins, `prepare_connection()` now accepts optional `database` and `datasette` arguments. +* 4th February 2020: [Datasette 0.35](https://docs.datasette.io/en/stable/changelog.html#v0-35) - new `.render_template()` method for plugins. +* 29th January 2020: [Datasette 0.34](https://docs.datasette.io/en/stable/changelog.html#v0-34) - improvements to search, `datasette publish cloudrun` and `datasette package`. +* 21st January 2020: [Deploying a data API using GitHub Actions and Cloud Run](https://simonwillison.net/2020/Jan/21/github-actions-cloud-run/) - how to use GitHub Actions and Google Cloud Run to automatically scrape data and deploy the result as an API with Datasette. +* 22nd December 2019: [Datasette 0.33](https://docs.datasette.io/en/stable/changelog.html#v0-33) - various small improvements. +* 19th December 2019: [Building tools to bring data-driven reporting to more newsrooms](https://medium.com/jsk-class-of-2020/building-tools-to-bring-data-driven-reporting-to-more-newsrooms-4520a0c9b3f2) - some notes on my JSK fellowship so far. +* 2nd December 2019: [Niche Museums](https://www.niche-museums.com/) is a new site entirely powered by Datasette, using custom templates and plugins. [niche-museums.com, powered by Datasette](https://simonwillison.net/2019/Nov/25/niche-museums/) describes how the site works, and [datasette-atom: Define an Atom feed using a custom SQL query](https://simonwillison.net/2019/Dec/3/datasette-atom/) describes how the new [datasette-atom plugin](https://github.com/simonw/datasette-atom) was used to add an Atom syndication feed to the site. +* 14th November 2019: [Datasette 0.32](https://docs.datasette.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function. +* 11th November 2019: [Datasette 0.31](https://docs.datasette.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. +* 18th October 2019: [Datasette 0.30](https://docs.datasette.io/en/stable/changelog.html#v0-30) +* 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. +* 7th July 2019: [Datasette 0.29](https://docs.datasette.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. * [datasette-cors](https://github.com/simonw/datasette-cors) - a plugin that lets you configure CORS access from a list of domains (or a set of domain wildcards) so you can make JavaScript calls to a Datasette instance from a specific set of other hosts. - * 23rd June 2019: [Porting Datasette to ASGI, and Turtles all the way down](https://simonwillison.net/2019/Jun/23/datasette-asgi/) - * 21st May 2019: The anonymized raw data from [the Stack Overflow Developer Survey 2019](https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/) has been [published in partnership with Glitch](https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/), powered by Datasette. - * 19th May 2019: [Datasette 0.28](https://docs.datasette.io/en/stable/changelog.html#v0-28) - a salmagundi of new features! +* 23rd June 2019: [Porting Datasette to ASGI, and Turtles all the way down](https://simonwillison.net/2019/Jun/23/datasette-asgi/) +* 21st May 2019: The anonymized raw data from [the Stack Overflow Developer Survey 2019](https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/) has been [published in partnership with Glitch](https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/), powered by Datasette. +* 19th May 2019: [Datasette 0.28](https://docs.datasette.io/en/stable/changelog.html#v0-28) - a salmagundi of new features! * No longer immutable! Datasette now supports [databases that change](https://docs.datasette.io/en/stable/changelog.html#supporting-databases-that-change). * [Faceting improvements](https://docs.datasette.io/en/stable/changelog.html#faceting-improvements-and-faceting-plugins) including facet-by-JSON-array and the ability to define custom faceting using plugins. * [datasette publish cloudrun](https://docs.datasette.io/en/stable/changelog.html#datasette-publish-cloudrun) lets you publish databases to Google's new Cloud Run hosting service. * New [register_output_renderer](https://docs.datasette.io/en/stable/changelog.html#register-output-renderer-plugins) plugin hook for adding custom output extensions to Datasette in addition to the default `.json` and `.csv`. * Dozens of other smaller features and tweaks - see [the release notes](https://docs.datasette.io/en/stable/changelog.html#v0-28) for full details. * Read more about this release here: [Datasette 0.28—and why master should always be releasable](https://simonwillison.net/2019/May/19/datasette-0-28/) - * 24th February 2019: [ +* 24th February 2019: [ sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) - a partner tool for easily creating SQLite databases for use with Datasette. - * 31st Janary 2019: [Datasette 0.27](https://docs.datasette.io/en/stable/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://docs.datasette.io/en/stable/ecosystem.html). - * 10th January 2019: [Datasette 0.26.1](https://docs.datasette.io/en/stable/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. - * 2nd January 2019: [Datasette 0.26](https://docs.datasette.io/en/stable/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. +* 31st Janary 2019: [Datasette 0.27](https://docs.datasette.io/en/stable/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://docs.datasette.io/en/stable/ecosystem.html). +* 10th January 2019: [Datasette 0.26.1](https://docs.datasette.io/en/stable/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. +* 2nd January 2019: [Datasette 0.26](https://docs.datasette.io/en/stable/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. * 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine. * 3rd October 2018: [The interesting ideas in Datasette](https://simonwillison.net/2018/Oct/4/datasette-ideas/) - a write-up of some of the less obvious interesting ideas embedded in the Datasette project. * 19th September 2018: [Datasette 0.25](https://docs.datasette.io/en/stable/changelog.html#v0-25) - New plugin hooks, improved database view support and an easier way to use more recent versions of SQLite. From 8ae0f9f7f0d644b0161165a1084f53acd2786f7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Dec 2020 12:16:13 -0800 Subject: [PATCH 0210/1866] Fixed spelling of Janary --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89245cf1..7861abbd 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly new * Read more about this release here: [Datasette 0.28—and why master should always be releasable](https://simonwillison.net/2019/May/19/datasette-0-28/) * 24th February 2019: [ sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) - a partner tool for easily creating SQLite databases for use with Datasette. -* 31st Janary 2019: [Datasette 0.27](https://docs.datasette.io/en/stable/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://docs.datasette.io/en/stable/ecosystem.html). +* 31st January 2019: [Datasette 0.27](https://docs.datasette.io/en/stable/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://docs.datasette.io/en/stable/ecosystem.html). * 10th January 2019: [Datasette 0.26.1](https://docs.datasette.io/en/stable/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. * 2nd January 2019: [Datasette 0.26](https://docs.datasette.io/en/stable/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. * 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine. From 4c25b035b2370983c8dd5e0c8762e9154e379774 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Dec 2020 14:41:03 -0800 Subject: [PATCH 0211/1866] arraynotcontains filter, closes #1132 --- datasette/filters.py | 11 ++++++++++- docs/json_api.rst | 7 ++++++- tests/test_api.py | 25 +++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 1524b32a..edf2de99 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -154,7 +154,16 @@ class Filters: where j.value = :{p} )""", '{c} contains "{v}"', - ) + ), + TemplatedFilter( + "arraynotcontains", + "array does not contain", + """rowid not in ( + select {t}.rowid from {t}, json_each({t}.{c}) j + where j.value = :{p} + )""", + '{c} does not contain "{v}"', + ), ] if detect_json1() else [] diff --git a/docs/json_api.rst b/docs/json_api.rst index 8d45ac6f..582a6159 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -267,7 +267,12 @@ You can filter the data returned by the table based on column values using a que Rows where column does not match any of the provided values. The inverse of ``__in=``. Also supports JSON arrays. ``?column__arraycontains=value`` - Works against columns that contain JSON arrays - matches if any of the values in that array match. + Works against columns that contain JSON arrays - matches if any of the values in that array match the provided value. + + This is only available if the ``json1`` SQLite extension is enabled. + +``?column__arraynotcontains=value`` + Works against columns that contain JSON arrays - matches if none of the values in that array match the provided value. This is only available if the ``json1`` SQLite extension is enabled. diff --git a/tests/test_api.py b/tests/test_api.py index 4339507c..a4c30414 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1121,7 +1121,7 @@ def test_table_filter_queries_multiple_of_same_type(app_client): @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") def test_table_filter_json_arraycontains(app_client): response = app_client.get("/fixtures/facetable.json?tags__arraycontains=tag1") - assert [ + assert response.json["rows"] == [ [ 1, "2019-01-14 08:00:00", @@ -1146,7 +1146,28 @@ def test_table_filter_json_arraycontains(app_client): "[]", "two", ], - ] == response.json["rows"] + ] + + +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +def test_table_filter_json_arraynotcontains(app_client): + response = app_client.get( + "/fixtures/facetable.json?tags__arraynotcontains=tag3&tags__not=[]" + ) + assert response.json["rows"] == [ + [ + 1, + "2019-01-14 08:00:00", + 1, + 1, + "CA", + 1, + "Mission", + '["tag1", "tag2"]', + '[{"foo": "bar"}]', + "one", + ] + ] def test_table_filter_extra_where(app_client): From fe86d853089f324f92daa950cc56f4052bf78f98 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Dec 2020 11:45:45 -0800 Subject: [PATCH 0212/1866] datasette serve --create option, closes #1135 --- datasette/cli.py | 21 ++++++++++++++++++++- docs/datasette-serve-help.txt | 1 + tests/test_cli.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/datasette/cli.py b/datasette/cli.py index e84695e3..32408d23 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -27,6 +27,7 @@ from .utils import ( StaticMount, ValueAsBooleanError, ) +from .utils.sqlite import sqlite3 from .utils.testing import TestClient from .version import __version__ @@ -299,7 +300,7 @@ def uninstall(packages, yes): @cli.command() -@click.argument("files", type=click.Path(exists=True), nargs=-1) +@click.argument("files", type=click.Path(), nargs=-1) @click.option( "-i", "--immutable", @@ -401,6 +402,11 @@ def uninstall(packages, yes): is_flag=True, help="Open Datasette in your web browser", ) +@click.option( + "--create", + is_flag=True, + help="Create database files if they do not exist", +) def serve( files, immutable, @@ -424,6 +430,7 @@ def serve( help_config, pdb, open_browser, + create, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -486,6 +493,18 @@ def serve( kwargs["config_dir"] = pathlib.Path(files[0]) files = [] + # Verify list of files, create if needed (and --create) + for file in files: + if not pathlib.Path(file).exists(): + if create: + sqlite3.connect(file).execute("vacuum") + else: + raise click.ClickException( + "Invalid value for '[FILES]...': Path '{}' does not exist.".format( + file + ) + ) + try: ds = Datasette(files, **kwargs) except SpatialiteNotFound: diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index bdaf0894..079ec9f8 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -40,4 +40,5 @@ Options: --help-config Show available config options --pdb Launch debugger on any errors -o, --open Open Datasette in your web browser + --create Create database files if they do not exist --help Show this message and exit. diff --git a/tests/test_cli.py b/tests/test_cli.py index a0ac7d7a..3f6b1840 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -146,6 +146,7 @@ def test_metadata_yaml(): help_config=False, pdb=False, open_browser=False, + create=False, return_instance=True, ) client = _TestClient(ds) @@ -221,3 +222,21 @@ def test_sql_errors_logged_to_stderr(ensure_eventloop): result = runner.invoke(cli, ["--get", "/:memory:.json?sql=select+blah"]) assert result.exit_code == 1 assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr + + +def test_serve_create(ensure_eventloop, tmpdir): + runner = CliRunner() + db_path = tmpdir / "does_not_exist_yet.db" + assert not db_path.exists() + result = runner.invoke( + cli, [str(db_path), "--create", "--get", "/-/databases.json"] + ) + assert result.exit_code == 0, result.output + databases = json.loads(result.output) + assert { + "name": "does_not_exist_yet", + "is_mutable": True, + "is_memory": False, + "hash": None, + }.items() <= databases[0].items() + assert db_path.exists() From 6000d1a724d0e28cdb102e7be83eac07a00b41e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Dec 2020 11:56:44 -0800 Subject: [PATCH 0213/1866] Fix for combining ?_search_x and ?_searchmode=raw, closes #1134 --- datasette/views/table.py | 4 +++- tests/test_api.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index a0de2a8e..3e9adf88 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -443,7 +443,9 @@ class TableView(RowTableShared): fts_table = fts_table or await db.fts_table(table) fts_pk = special_args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) search_args = dict( - pair for pair in special_args.items() if pair[0].startswith("_search") + pair + for pair in special_args.items() + if pair[0].startswith("_search") and pair[0] != "_searchmode" ) search = "" search_mode_raw = special_args.get("_searchmode") == "raw" diff --git a/tests/test_api.py b/tests/test_api.py index a4c30414..10755b95 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1035,6 +1035,13 @@ def test_sortable_columns_metadata(app_client): [2, "terry dog", "sara weasel", "puma"], ], ), + ( + # _searchmode=raw combined with _search_COLUMN + "/fixtures/searchable.json?_search_text2=te*&_searchmode=raw", + [ + [1, "barry cat", "terry dog", "panther"], + ], + ), ( "/fixtures/searchable.json?_search=weasel", [[2, "terry dog", "sara weasel", "puma"]], From 387b471b88788069191bc845224b7712d92e9c0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Dec 2020 12:03:44 -0800 Subject: [PATCH 0214/1866] Release 0.52.5 Refs #1134 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index ce06fe1d..b0a59018 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52.4" +__version__ = "0.52.5" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 86d844f7..c79e7c86 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_52_5: + +0.52.5 (2020-12-09) +------------------- + +- Fix for error caused by combining the ``_searchmode=raw`` and ``?_search_COLUMN`` parameters. (`#1134 `__) + .. _v0_52_4: 0.52.4 (2020-12-05) From 4c6407cd74070237fdad0dd6df4d016740806fbd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Dec 2020 12:14:33 -0800 Subject: [PATCH 0215/1866] Releasing bug fixes from a branch, closes #1136 --- docs/contributing.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ca194001..8cd9c210 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -204,6 +204,34 @@ You are welcome to try these out, but please be aware that details may change be Please join `discussions on the issue tracker `__ to share your thoughts and experiences with on alpha and beta features that you try out. +.. _contributing_bug_fix_branch: + +Releasing bug fixes from a branch +--------------------------------- + +If it's necessary to publish a bug fix release without shipping new features that have landed on ``main`` a release branch can be used. + +Create it from the relevant last tagged release like so:: + + git branch 0.52.x 0.52.4 + git checkout 0.52.x + +Next cherry-pick the commits containing the bug fixes:: + + git cherry-pick COMMIT + +Write the release notes in the branch, and update the version number in ``version.py``. Then push the branch:: + + git push -u origin 0.52.x + +Once the tests have completed, publish the release from that branch target using the GitHub `Draft a new release `__ form. + +Finally, cherry-pick the commit with the release notes and version number bump across to ``main``:: + + git checkout main + git cherry-pick COMMIT + git push + .. _contributing_upgrading_codemirror: Upgrading CodeMirror From e0b54d09115ded459e09e2e89e0962cfddcb0244 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 15:20:43 -0800 Subject: [PATCH 0216/1866] No longer using Wiki for examples --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7861abbd..71e488f7 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover [Explore a demo](https://fivethirtyeight.datasettes.com/fivethirtyeight), watch [a video about the project](https://www.youtube.com/watch?v=pTr1uLQTJNE) or try it out by [uploading and publishing your own CSV data](https://simonwillison.net/2019/Apr/23/datasette-glitch/). * Comprehensive documentation: https://docs.datasette.io/ -* Examples: https://github.com/simonw/datasette/wiki/Datasettes +* Examples: https://datasette.io/examples * Live demo of current main: https://latest.datasette.io/ * Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) diff --git a/docs/index.rst b/docs/index.rst index ff8db04b..eafc5bdb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover `Explore a demo `__, watch `a presentation about the project `__ or :ref:`getting_started_glitch`. -More examples: https://github.com/simonw/datasette/wiki/Datasettes +More examples: https://datasette.io/examples Support questions, feedback? Join our `GitHub Discussions forum `__. From 7ef80d0145dc9a2a16c46823704517d7f35fbe45 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 15:24:16 -0800 Subject: [PATCH 0217/1866] News is now on datasette.io/news Closes #1137, closes #659 --- README.md | 83 ++++++------------------------------------------------- 1 file changed, 8 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 71e488f7..16fc8f0e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover [Explore a demo](https://fivethirtyeight.datasettes.com/fivethirtyeight), watch [a video about the project](https://www.youtube.com/watch?v=pTr1uLQTJNE) or try it out by [uploading and publishing your own CSV data](https://simonwillison.net/2019/Apr/23/datasette-glitch/). +* Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples * Live demo of current main: https://latest.datasette.io/ @@ -23,83 +24,15 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover Want to stay up-to-date with the project? Subscribe to the [Datasette Weekly newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. -## News - -* 28th November 2020: [Datasette 0.52](https://docs.datasette.io/en/stable/changelog.html#v0-52) - `--config` is now `--setting`, new `database_actions` plugin hook, `datasette publish cloudrun --apt-get-install` option and several bug fixes. -* 31st October 2020: [Datasette 0.51](https://docs.datasette.io/en/stable/changelog.html#v0-51) - A new visual design, plugin hooks for adding navigation options, better handling of binary data, URL building utility methods and better support for running Datasette behind a proxy. [Annotated release notes](https://simonwillison.net/2020/Nov/1/datasette-0-51/). -* 9th October 2020: [Datasette 0.50](https://docs.datasette.io/en/stable/changelog.html#v0-50) - New column actions menu. `datasette.client` object for plugins to make internal API requests. Improved documentation on deploying Datasette. [Annotated release notes](https://simonwillison.net/2020/Oct/9/datasette-0-50/). -* 14th September 2020: [Datasette 0.49](https://docs.datasette.io/en/stable/changelog.html#v0-49) - JSON API for writable canned queries, path parameters for custom pages. See also [Datasette 0.49: The annotated release notes](https://simonwillison.net/2020/Sep/15/datasette-0-49/). -* 16th August 2020: [Datasette 0.48](https://docs.datasette.io/en/stable/changelog.html#v0-48) - Documentation now lives at [docs.datasette.io](https://docs.datasette.io/), improvements to the `extra_template_vars`, `extra_css_urls`, `extra_js_urls` and `extra_body_script` plugin hooks. -* 11th August 2020: [Datasette 0.47](https://docs.datasette.io/en/stable/changelog.html#v0-47) - Datasette can now be installed using Homebrew! `brew install simonw/datasette/datasette`. Also new: `datasette install name-of-plugin` and `datasette uninstall name-of-plugin` commands, and `datasette --get '/-/versions.json'` to output the result of Datasette HTTP calls on the command-line. -* 9th August 2020: [Datasette 0.46](https://docs.datasette.io/en/stable/changelog.html#v0-46) - security fix relating to CSRF protection for writable canned queries, a new logo, new debugging tools, improved file downloads and more. -* 6th August 2020: [GraphQL in Datasette with the new datasette-graphql plugin](https://simonwillison.net/2020/Aug/7/datasette-graphql/) -* 24th July 2020: Two new plugins: [datasette-copyable and datasette-insert-api](https://simonwillison.net/2020/Jul/23/datasette-copyable-datasette-insert-api/). `datasette-copyable` adds copy-and-paste export options, and `datasette-insert-api` lets you create tables and insert or update data by POSTing JSON directly to Datasette. -* 1st July 2020: [Datasette 0.45](https://docs.datasette.io/en/stable/changelog.html#v0-45) - [Magic parameters for canned queries](https://docs.datasette.io/en/stable/sql_queries.html#canned-queries-magic-parameters), a log out feature, improved plugin documentation and four new plugin hooks. See also [Datasette 0.45: The annotated release notes](https://simonwillison.net/2020/Jul/1/datasette-045/). -* 20th June 2020: [A cookiecutter template for writing Datasette plugins](https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/) -* 11th June 2020: [Datasette 0.44](https://docs.datasette.io/en/stable/changelog.html#v0-44) - [Authentication and permissions](https://docs.datasette.io/en/stable/authentication.html), [writable canned queries](https://docs.datasette.io/en/stable/sql_queries.html#writable-canned-queries), flash messages, new plugin hooks and much, much more. -* 28th May 2020: [Datasette 0.43](https://docs.datasette.io/en/stable/changelog.html#v0-43) - Redesigned [register_output_renderer](https://docs.datasette.io/en/stable/plugins.html#plugin-register-output-renderer) plugin hook and various small improvements and fixes. -* 8th May 2020: [Datasette 0.42](https://docs.datasette.io/en/stable/changelog.html#v0-42) - Documented internal methods for plugins to execute read queries against a database. -* 6th May 2020: [Datasette 0.41](https://docs.datasette.io/en/stable/changelog.html#v0-41) - New mechanism for [creating custom pages](https://docs.datasette.io/en/0.41/custom_templates.html#custom-pages), new [configuration directory mode](https://docs.datasette.io/en/0.41/config.html#configuration-directory-mode), new `?column__notlike=` table filter and various other smaller improvements. -* 21st April 2020: [Datasette 0.40](https://docs.datasette.io/en/stable/changelog.html#v0-40) - Metadata can now be provided as YAML instead of JSON. Publishing to Zeit Now v1 is no longer supported, but Now v2 support is provided by the new [datasette-publish-now](https://github.com/simonw/datasette-publish-now) plugin. Various bug fixes. -* 24th March 2020: [Datasette 0.39](https://docs.datasette.io/en/stable/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table. -* 8th March 2020: [Datasette 0.38](https://docs.datasette.io/en/stable/changelog.html#v0-38) - New `--memory` option for `datasete publish cloudrun`, [Docker image](https://hub.docker.com/r/datasetteproject/datasette) upgraded to SQLite 3.31.1. -* 25th February 2020: [Datasette 0.37](https://docs.datasette.io/en/stable/changelog.html#v0-37) - new internal APIs enabling plugins to safely write to databases. Read more here: [Datasette Writes](https://simonwillison.net/2020/Feb/26/weeknotes-datasette-writes/). -* 21st February 2020: [Datasette 0.36](https://docs.datasette.io/en/stable/changelog.html#v0-36) - new internals documentation for plugins, `prepare_connection()` now accepts optional `database` and `datasette` arguments. -* 4th February 2020: [Datasette 0.35](https://docs.datasette.io/en/stable/changelog.html#v0-35) - new `.render_template()` method for plugins. -* 29th January 2020: [Datasette 0.34](https://docs.datasette.io/en/stable/changelog.html#v0-34) - improvements to search, `datasette publish cloudrun` and `datasette package`. -* 21st January 2020: [Deploying a data API using GitHub Actions and Cloud Run](https://simonwillison.net/2020/Jan/21/github-actions-cloud-run/) - how to use GitHub Actions and Google Cloud Run to automatically scrape data and deploy the result as an API with Datasette. -* 22nd December 2019: [Datasette 0.33](https://docs.datasette.io/en/stable/changelog.html#v0-33) - various small improvements. -* 19th December 2019: [Building tools to bring data-driven reporting to more newsrooms](https://medium.com/jsk-class-of-2020/building-tools-to-bring-data-driven-reporting-to-more-newsrooms-4520a0c9b3f2) - some notes on my JSK fellowship so far. -* 2nd December 2019: [Niche Museums](https://www.niche-museums.com/) is a new site entirely powered by Datasette, using custom templates and plugins. [niche-museums.com, powered by Datasette](https://simonwillison.net/2019/Nov/25/niche-museums/) describes how the site works, and [datasette-atom: Define an Atom feed using a custom SQL query](https://simonwillison.net/2019/Dec/3/datasette-atom/) describes how the new [datasette-atom plugin](https://github.com/simonw/datasette-atom) was used to add an Atom syndication feed to the site. -* 14th November 2019: [Datasette 0.32](https://docs.datasette.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function. -* 11th November 2019: [Datasette 0.31](https://docs.datasette.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5. -* 18th October 2019: [Datasette 0.30](https://docs.datasette.io/en/stable/changelog.html#v0-30) -* 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail. -* 7th July 2019: [Datasette 0.29](https://docs.datasette.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more... - * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams. - * [datasette-cors](https://github.com/simonw/datasette-cors) - a plugin that lets you configure CORS access from a list of domains (or a set of domain wildcards) so you can make JavaScript calls to a Datasette instance from a specific set of other hosts. -* 23rd June 2019: [Porting Datasette to ASGI, and Turtles all the way down](https://simonwillison.net/2019/Jun/23/datasette-asgi/) -* 21st May 2019: The anonymized raw data from [the Stack Overflow Developer Survey 2019](https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/) has been [published in partnership with Glitch](https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/), powered by Datasette. -* 19th May 2019: [Datasette 0.28](https://docs.datasette.io/en/stable/changelog.html#v0-28) - a salmagundi of new features! - * No longer immutable! Datasette now supports [databases that change](https://docs.datasette.io/en/stable/changelog.html#supporting-databases-that-change). - * [Faceting improvements](https://docs.datasette.io/en/stable/changelog.html#faceting-improvements-and-faceting-plugins) including facet-by-JSON-array and the ability to define custom faceting using plugins. - * [datasette publish cloudrun](https://docs.datasette.io/en/stable/changelog.html#datasette-publish-cloudrun) lets you publish databases to Google's new Cloud Run hosting service. - * New [register_output_renderer](https://docs.datasette.io/en/stable/changelog.html#register-output-renderer-plugins) plugin hook for adding custom output extensions to Datasette in addition to the default `.json` and `.csv`. - * Dozens of other smaller features and tweaks - see [the release notes](https://docs.datasette.io/en/stable/changelog.html#v0-28) for full details. - * Read more about this release here: [Datasette 0.28—and why master should always be releasable](https://simonwillison.net/2019/May/19/datasette-0-28/) -* 24th February 2019: [ -sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) - a partner tool for easily creating SQLite databases for use with Datasette. -* 31st January 2019: [Datasette 0.27](https://docs.datasette.io/en/stable/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://docs.datasette.io/en/stable/ecosystem.html). -* 10th January 2019: [Datasette 0.26.1](https://docs.datasette.io/en/stable/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options. -* 2nd January 2019: [Datasette 0.26](https://docs.datasette.io/en/stable/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument. -* 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine. -* 3rd October 2018: [The interesting ideas in Datasette](https://simonwillison.net/2018/Oct/4/datasette-ideas/) - a write-up of some of the less obvious interesting ideas embedded in the Datasette project. -* 19th September 2018: [Datasette 0.25](https://docs.datasette.io/en/stable/changelog.html#v0-25) - New plugin hooks, improved database view support and an easier way to use more recent versions of SQLite. -* 23rd July 2018: [Datasette 0.24](https://docs.datasette.io/en/stable/changelog.html#v0-24) - a number of small new features -* 29th June 2018: [datasette-vega](https://github.com/simonw/datasette-vega), a new plugin for visualizing data as bar, line or scatter charts -* 21st June 2018: [Datasette 0.23.1](https://docs.datasette.io/en/stable/changelog.html#v0-23-1) - minor bug fixes -* 18th June 2018: [Datasette 0.23: CSV, SpatiaLite and more](https://docs.datasette.io/en/stable/changelog.html#v0-23) - CSV export, foreign key expansion in JSON and CSV, new config options, improved support for SpatiaLite and a bunch of other improvements -* 23rd May 2018: [Datasette 0.22.1 bugfix](https://github.com/simonw/datasette/releases/tag/0.22.1) plus we now use [versioneer](https://github.com/warner/python-versioneer) -* 20th May 2018: [Datasette 0.22: Datasette Facets](https://simonwillison.net/2018/May/20/datasette-facets) -* 5th May 2018: [Datasette 0.21: New _shape=, new _size=, search within columns](https://github.com/simonw/datasette/releases/tag/0.21) -* 25th April 2018: [Exploring the UK Register of Members Interests with SQL and Datasette](https://simonwillison.net/2018/Apr/25/register-members-interests/) - a tutorial describing how [register-of-members-interests.datasettes.com](https://register-of-members-interests.datasettes.com/) was built ([source code here](https://github.com/simonw/register-of-members-interests)) -* 20th April 2018: [Datasette plugins, and building a clustered map visualization](https://simonwillison.net/2018/Apr/20/datasette-plugins/) - introducing Datasette's new plugin system and [datasette-cluster-map](https://pypi.org/project/datasette-cluster-map/), a plugin for visualizing data on a map -* 20th April 2018: [Datasette 0.20: static assets and templates for plugins](https://github.com/simonw/datasette/releases/tag/0.20) -* 16th April 2018: [Datasette 0.19: plugins preview](https://github.com/simonw/datasette/releases/tag/0.19) -* 14th April 2018: [Datasette 0.18: units](https://github.com/simonw/datasette/releases/tag/0.18) -* 9th April 2018: [Datasette 0.15: sort by column](https://github.com/simonw/datasette/releases/tag/0.15) -* 28th March 2018: [Baltimore Sun Public Salary Records](https://simonwillison.net/2018/Mar/28/datasette-in-the-wild/) - a data journalism project from the Baltimore Sun powered by Datasette - source code [is available here](https://github.com/baltimore-sun-data/salaries-datasette) -* 27th March 2018: [Cloud-first: Rapid webapp deployment using containers](https://wwwf.imperial.ac.uk/blog/research-software-engineering/2018/03/27/cloud-first-rapid-webapp-deployment-using-containers/) - a tutorial covering deploying Datasette using Microsoft Azure by the Research Software Engineering team at Imperial College London -* 28th January 2018: [Analyzing my Twitter followers with Datasette](https://simonwillison.net/2018/Jan/28/analyzing-my-twitter-followers/) - a tutorial on using Datasette to analyze follower data pulled from the Twitter API -* 17th January 2018: [Datasette Publish: a web app for publishing CSV files as an online database](https://simonwillison.net/2018/Jan/17/datasette-publish/) -* 12th December 2017: [Building a location to time zone API with SpatiaLite, OpenStreetMap and Datasette](https://simonwillison.net/2017/Dec/12/building-a-location-time-zone-api/) -* 9th December 2017: [Datasette 0.14: customization edition](https://github.com/simonw/datasette/releases/tag/0.14) -* 25th November 2017: [New in Datasette: filters, foreign keys and search](https://simonwillison.net/2017/Nov/25/new-in-datasette/) -* 13th November 2017: [Datasette: instantly create and publish an API for your SQLite databases](https://simonwillison.net/2017/Nov/13/datasette/) - ## Installation - pip3 install datasette +If you are on a Mac, [Homebrew](https://brew.sh/) is the easiest way to install Datasette: + + brew install datasette + +You can also install it using `pip` or `pipx`: + + pip install datasette Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. From 2c0aca4887ed65167606a5fd084f35d046e2a00a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 15:28:44 -0800 Subject: [PATCH 0218/1866] _header=off option for CSV export, closes #1133 --- datasette/views/base.py | 3 ++- docs/csv_export.rst | 16 ++++++++++++++++ tests/test_csv.py | 8 ++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index b8860b74..76e03206 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -307,7 +307,8 @@ class DataView(BaseView): if not first: data, _, _ = await self.data(request, database, hash, **kwargs) if first: - await writer.writerow(headings) + if request.args.get("_header") != "off": + await writer.writerow(headings) first = False next = data.get("next") for row in data["rows"]: diff --git a/docs/csv_export.rst b/docs/csv_export.rst index 0bda20ef..7f0d8396 100644 --- a/docs/csv_export.rst +++ b/docs/csv_export.rst @@ -28,6 +28,22 @@ file, which looks like this and has the following options: You can try that out on https://latest.datasette.io/fixtures/facetable?_size=4 +.. _csv_export_url_parameters: + +URL parameters +-------------- + +The following options can be used to customize the CSVs returned by Datasette. + +``?_header=off`` + This removes the first row of the CSV file specifying the headings - only the row data will be returned. + +``?_stream=on`` + Stream all matching records, not just the first page of results. See below. + +``?_dl=on`` + Causes Datasette to return a ``content-disposition: attachment; filename="filename.csv"`` header. + Streaming all records --------------------- diff --git a/tests/test_csv.py b/tests/test_csv.py index 0fd665a9..6b17033c 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -64,6 +64,14 @@ def test_table_csv_cors_headers(app_client_with_cors): assert "*" == response.headers["Access-Control-Allow-Origin"] +def test_table_csv_no_header(app_client): + response = app_client.get("/fixtures/simple_primary_key.csv?_header=off") + assert response.status == 200 + assert not response.headers.get("Access-Control-Allow-Origin") + assert "text/plain; charset=utf-8" == response.headers["content-type"] + assert EXPECTED_TABLE_CSV.split("\r\n", 1)[1] == response.text + + def test_table_csv_with_labels(app_client): response = app_client.get("/fixtures/facetable.csv?_labels=1") assert response.status == 200 From 967cc05545480f09d421a7bf8b6dbfc27609a181 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 15:37:08 -0800 Subject: [PATCH 0219/1866] Powered by links to datasette.io, closes #1138 --- datasette/templates/_footer.html | 2 +- datasette/templates/patterns.html | 2 +- setup.py | 2 +- tests/test_html.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datasette/templates/_footer.html b/datasette/templates/_footer.html index f930f445..b1380ae9 100644 --- a/datasette/templates/_footer.html +++ b/datasette/templates/_footer.html @@ -1,4 +1,4 @@ -Powered by Datasette +Powered by Datasette {% if query_ms %}· Query took {{ query_ms|round(3) }}ms{% endif %} {% if metadata %} {% if metadata.license or metadata.license_url %}· Data license: diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 4ef2c29f..984c1bf6 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -476,7 +476,7 @@

    .ft

    -
    Powered by Datasette +
    Powered by Datasette · Data license: Apache License 2.0 · diff --git a/setup.py b/setup.py index 82696b38..e9eb1597 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( long_description_content_type="text/markdown", author="Simon Willison", license="Apache License, Version 2.0", - url="https://github.com/simonw/datasette", + url="https://datasette.io/", project_urls={ "Documentation": "https://docs.datasette.io/en/stable/", "Changelog": "https://docs.datasette.io/en/stable/changelog.html", diff --git a/tests/test_html.py b/tests/test_html.py index b9d3afcd..8b0b1c8d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1079,7 +1079,7 @@ def assert_footer_links(soup): assert "tests/fixtures.py" == source_link.text.strip() assert "Apache License 2.0" == license_link.text.strip() assert "About Datasette" == about_link.text.strip() - assert "https://github.com/simonw/datasette" == datasette_link["href"] + assert "https://datasette.io/" == datasette_link["href"] assert ( "https://github.com/simonw/datasette/blob/master/tests/fixtures.py" == source_link["href"] @@ -1461,7 +1461,7 @@ def test_base_url_config(app_client_base_url_prefix, path): not href.startswith("#") and href not in { - "https://github.com/simonw/datasette", + "https://datasette.io/", "https://github.com/simonw/datasette/blob/master/LICENSE", "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", "/login-as-root", # Only used for the latest.datasette.io demo From 02bb373194000d2b15f61914e7c5fdb124275bcd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 17:38:16 -0800 Subject: [PATCH 0220/1866] Updated release process --- docs/contributing.rst | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 8cd9c210..24d5c8f0 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -181,17 +181,9 @@ You can generate the list of issue references for a specific release by pasting ), ].sort().join(", "); -For non-bugfix releases you may want to update the news section of ``README.md`` as part of the same commit. +To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. -To tag and push the releaes, run the following:: - - git tag 0.25.2 - git push --tags - -Final steps once the release has deployed to https://pypi.org/project/datasette/ - -* Manually post the new release to GitHub releases: https://github.com/simonw/datasette/releases - you can convert the release notes to Markdown by copying and pasting the rendered HTML into this tool: https://euangoddard.github.io/clipboard2markdown/ -* Manually kick off a build of the `stable` branch on Read The Docs: https://readthedocs.org/projects/datasette/builds/ +Finally, post a news item about the release on `datasette.io `__ by editing the `news.yaml `__ file in that site's repository. .. _contributing_alpha_beta: From 0c616f732cee79db80cad830917666f41b344262 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 10 Dec 2020 17:44:36 -0800 Subject: [PATCH 0221/1866] Release 0.53 Refs #1132, #1135, #1133, #1138, #1137 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b0a59018..a5edecfa 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.52.5" +__version__ = "0.53" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index c79e7c86..c570642f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_53: + +0.53 (2020-12-10) +----------------- + +Datasette has an official project website now, at https://datasette.io/. This release mainly updates the documentation to reflect the new site. + +- New ``?column__arraynotcontains=`` table filter. (`#1132 `__) +- ``datasette serve`` has a new ``--create`` option, which will create blank database files if they do not already exist rather than exiting with an error. (`#1135 `__) +- New ``?_header=off`` option for CSV export which omits the CSV header row, :ref:`documented here `. (`#1133 `__) +- "Powered by Datasette" link in the footer now links to https://datasette.io/. (`#1138 `__) +- Project news no longer lives in the README - it can now be found at https://datasette.io/news. (`#1137 `__) + + .. _v0_52_5: 0.52.5 (2020-12-09) From 6119bd797366a899119f1bba51c1c8cba2efc8fc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Dec 2020 13:44:39 -0800 Subject: [PATCH 0222/1866] Update pytest requirement from <6.2.0,>=5.2.2 to >=5.2.2,<6.3.0 (#1145) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.2.2...6.2.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e9eb1597..be94c1c6 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( extras_require={ "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ - "pytest>=5.2.2,<6.2.0", + "pytest>=5.2.2,<6.3.0", "pytest-asyncio>=0.10,<0.15", "beautifulsoup4>=4.8.1,<4.10.0", "black==20.8b1", From 5e9895c67f08e9f42acedd3d6d29512ac446e15f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 17 Dec 2020 17:01:18 -0800 Subject: [PATCH 0223/1866] Database(memory_name=) for shared in-memory databases, closes #1151 --- datasette/database.py | 24 +++++++++++++++++++-- docs/internals.rst | 37 +++++++++++++++++++++++++++++--- tests/test_internals_database.py | 30 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 412e0c59..a977b362 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -24,11 +24,18 @@ connections = threading.local() class Database: - def __init__(self, ds, path=None, is_mutable=False, is_memory=False): + def __init__( + self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + ): self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory + self.memory_name = memory_name + if memory_name is not None: + self.path = memory_name + self.is_memory = True + self.is_mutable = True self.hash = None self.cached_size = None self.cached_table_counts = None @@ -46,6 +53,16 @@ class Database: } def connect(self, write=False): + if self.memory_name: + uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) + conn = sqlite3.connect( + uri, + uri=True, + check_same_thread=False, + ) + if not write: + conn.execute("PRAGMA query_only=1") + return conn if self.is_memory: return sqlite3.connect(":memory:") # mode=ro or immutable=1? @@ -215,7 +232,10 @@ class Database: @property def name(self): if self.is_memory: - return ":memory:" + if self.memory_name: + return ":memory:{}".format(self.memory_name) + else: + return ":memory:" else: return Path(self.path).stem diff --git a/docs/internals.rst b/docs/internals.rst index ff566f69..b68a1d8a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -270,11 +270,16 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` This will add a mutable database from the provided file path. -The ``Database()`` constructor takes four arguments: the first is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. +To create a shared in-memory database named ``statistics``, use the following: -Use ``is_mutable`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. +.. code-block:: python -Use ``is_memory`` if the connection is to an in-memory SQLite database. + from datasette.database import Database + + datasette.add_database("statistics", Database( + datasette, + memory_name="statistics" + )) .. _datasette_remove_database: @@ -480,6 +485,32 @@ Database class Instances of the ``Database`` class can be used to execute queries against attached SQLite databases, and to run introspection against their schemas. +.. _database_constructor: + +Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) +---------------------------------------------------------------------------- + +The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. + +The arguments are as follows: + +``ds`` - :ref:`internals_datasette` (required) + The Datasette instance you are attaching this database to. + +``path`` - string + Path to a SQLite database file on disk. + +``is_mutable`` - boolean + Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + +``is_memory`` - boolean + Use this to create non-shared memory connections. + +``memory_name`` - string or ``None`` + Use this to create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads and will persist an changes made to them for the lifetime of the Datasette server process. + +The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. + .. _database_execute: await db.execute(sql, ...) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 49b8a1b3..dc1af48c 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -464,3 +464,33 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False + + +@pytest.mark.asyncio +async def test_database_memory_name(app_client): + ds = app_client.ds + foo1 = Database(ds, memory_name="foo") + foo2 = Database(ds, memory_name="foo") + bar1 = Database(ds, memory_name="bar") + bar2 = Database(ds, memory_name="bar") + for db in (foo1, foo2, bar1, bar2): + table_names = await db.table_names() + assert table_names == [] + # Now create a table in foo + await foo1.execute_write("create table foo (t text)", block=True) + assert await foo1.table_names() == ["foo"] + assert await foo2.table_names() == ["foo"] + assert await bar1.table_names() == [] + assert await bar2.table_names() == [] + + +@pytest.mark.asyncio +async def test_in_memory_databases_forbid_writes(app_client): + ds = app_client.ds + db = Database(ds, memory_name="test") + with pytest.raises(sqlite3.OperationalError): + await db.execute("create table foo (t text)") + assert await db.table_names() == [] + # Using db.execute_write() should work: + await db.execute_write("create table foo (t text)", block=True) + assert await db.table_names() == ["foo"] From ebc7aa287c99fe6114b79aeab8efb8d4489a6182 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 18 Dec 2020 14:34:05 -0800 Subject: [PATCH 0224/1866] In-memory _schemas database tracking schemas of attached tables, closes #1150 --- datasette/app.py | 39 +++++++- datasette/cli.py | 3 + datasette/default_permissions.py | 2 + datasette/utils/__init__.py | 7 +- datasette/utils/schemas.py | 162 +++++++++++++++++++++++++++++++ datasette/views/base.py | 2 + tests/test_plugins.py | 2 +- tests/test_schemas.py | 68 +++++++++++++ 8 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 datasette/utils/schemas.py create mode 100644 tests/test_schemas.py diff --git a/datasette/app.py b/datasette/app.py index 9bc84df0..cc8506e2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) +from .utils.schemas import init_schemas, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, @@ -222,6 +223,11 @@ class Datasette: elif memory: self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() + # memory_name is a random string so that each Datasette instance gets its own + # unique in-memory named database - otherwise unit tests can fail with weird + # errors when different instances accidentally share an in-memory database + self.add_database("_schemas", Database(self, memory_name=secrets.token_hex())) + self._schemas_created = False for file in self.files: path = file is_memory = False @@ -326,6 +332,33 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + async def refresh_schemas(self): + schema_db = self.databases["_schemas"] + if not self._schemas_created: + await init_schemas(schema_db) + self._schemas_created = True + + current_schema_versions = { + row["database_name"]: row["schema_version"] + for row in await schema_db.execute( + "select database_name, schema_version from databases" + ) + } + for database_name, db in self.databases.items(): + schema_version = (await db.execute("PRAGMA schema_version")).first()[0] + # Compare schema versions to see if we should skip it + if schema_version == current_schema_versions.get(database_name): + continue + await schema_db.execute_write( + """ + INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + VALUES (?, ?, ?, ?) + """, + [database_name, db.path, db.is_memory, schema_version], + block=True, + ) + await populate_schema_tables(schema_db, db) + @property def urls(self): return Urls(self) @@ -342,7 +375,8 @@ class Datasette: def get_database(self, name=None): if name is None: - return next(iter(self.databases.values())) + # Return first no-_schemas database + name = [key for key in self.databases.keys() if key != "_schemas"][0] return self.databases[name] def add_database(self, name, db): @@ -590,7 +624,8 @@ class Datasette: "is_memory": d.is_memory, "hash": d.hash, } - for d in sorted(self.databases.values(), key=lambda d: d.name) + for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) + if name != "_schemas" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 32408d23..50367fb3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -134,6 +134,9 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): + if name == "_schemas": + # Don't include the in-memory _schemas database + continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 9f1d9c62..62cab83a 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,6 +13,8 @@ def permission_allowed(datasette, actor, action, resource): if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": + if resource == "_schemas" and (actor is None or actor.get("id") != "root"): + return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2576090a..ac1d82f7 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1024,11 +1024,12 @@ def find_spatialite(): async def initial_path_for_datasette(datasette): "Return suggested path for opening this Datasette, based on number of DBs and tables" - if len(datasette.databases) == 1: - db_name = next(iter(datasette.databases.keys())) + databases = dict([p for p in datasette.databases.items() if p[0] != "_schemas"]) + if len(databases) == 1: + db_name = next(iter(databases.keys())) path = datasette.urls.database(db_name) # Does this DB only have one table? - db = next(iter(datasette.databases.values())) + db = next(iter(databases.values())) tables = await db.table_names() if len(tables) == 1: path = datasette.urls.table(db_name, tables[0]) diff --git a/datasette/utils/schemas.py b/datasette/utils/schemas.py new file mode 100644 index 00000000..4612e236 --- /dev/null +++ b/datasette/utils/schemas.py @@ -0,0 +1,162 @@ +async def init_schemas(db): + await db.execute_write( + """ + CREATE TABLE databases ( + "database_name" TEXT PRIMARY KEY, + "path" TEXT, + "is_memory" INTEGER, + "schema_version" INTEGER + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE tables ( + "database_name" TEXT, + "table_name" TEXT, + "rootpage" INTEGER, + "sql" TEXT, + PRIMARY KEY (database_name, table_name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE columns ( + "database_name" TEXT, + "table_name" TEXT, + "cid" INTEGER, + "name" TEXT, + "type" TEXT, + "notnull" INTEGER, + "default_value" TEXT, -- renamed from dflt_value + "is_pk" INTEGER, -- renamed from pk + "hidden" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE indexes ( + "database_name" TEXT, + "table_name" TEXT, + "seq" INTEGER, + "name" TEXT, + "unique" INTEGER, + "origin" TEXT, + "partial" INTEGER, + PRIMARY KEY (database_name, table_name, name) + ) + """, + block=True, + ) + await db.execute_write( + """ + CREATE TABLE foreign_keys ( + "database_name" TEXT, + "table_name" TEXT, + "id" INTEGER, + "seq" INTEGER, + "table" TEXT, + "from" TEXT, + "to" TEXT, + "on_update" TEXT, + "on_delete" TEXT, + "match" TEXT + ) + """, + block=True, + ) + + +async def populate_schema_tables(schema_db, db): + database_name = db.name + await schema_db.execute_write( + "delete from tables where database_name = ?", [database_name], block=True + ) + tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows + for table in tables: + table_name = table["name"] + await schema_db.execute_write( + """ + insert into tables (database_name, table_name, rootpage, sql) + values (?, ?, ?, ?) + """, + [database_name, table_name, table["rootpage"], table["sql"]], + block=True, + ) + # And the columns + await schema_db.execute_write( + "delete from columns where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + columns = await db.table_column_details(table_name) + for column in columns: + params = { + **{"database_name": database_name, "table_name": table_name}, + **column._asdict(), + } + await schema_db.execute_write( + """ + insert into columns ( + database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden + ) VALUES ( + :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden + ) + """, + params, + block=True, + ) + # And the foreign_keys + await schema_db.execute_write( + "delete from foreign_keys where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + foreign_keys = ( + await db.execute(f"PRAGMA foreign_key_list([{table_name}])") + ).rows + for foreign_key in foreign_keys: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(foreign_key), + } + await schema_db.execute_write( + """ + insert into foreign_keys ( + database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match + ) VALUES ( + :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match + ) + """, + params, + block=True, + ) + # And the indexes + await schema_db.execute_write( + "delete from indexes where database_name = ? and table_name = ?", + [database_name, table_name], + block=True, + ) + indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows + for index in indexes: + params = { + **{"database_name": database_name, "table_name": table_name}, + **dict(index), + } + await schema_db.execute_write( + """ + insert into indexes ( + database_name, table_name, seq, name, "unique", origin, partial + ) VALUES ( + :database_name, :table_name, :seq, :name, :unique, :origin, :partial + ) + """, + params, + block=True, + ) diff --git a/datasette/views/base.py b/datasette/views/base.py index 76e03206..73bf9459 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -115,6 +115,8 @@ class BaseView: return Response.text("Method not allowed", status=405) async def dispatch_request(self, request, *args, **kwargs): + if self.ds: + await self.ds.refresh_schemas() handler = getattr(self, request.method.lower(), None) return await handler(request, *args, **kwargs) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 93b444ab..61e7d4b5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -293,7 +293,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): def test_hook_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "fixtures" == response.headers["x-databases"] + assert "_schemas, fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 00000000..87656784 --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,68 @@ +from .fixtures import app_client +import pytest + + +def test_schemas_only_available_to_root(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + assert app_client.get("/_schemas").status == 403 + assert app_client.get("/_schemas", cookies={"ds_actor": cookie}).status == 200 + + +def test_schemas_databases(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + databases = app_client.get( + "/_schemas/databases.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(databases) == 2 + assert databases[0]["database_name"] == "_schemas" + assert databases[1]["database_name"] == "fixtures" + + +def test_schemas_tables(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + tables = app_client.get( + "/_schemas/tables.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(tables) > 5 + table = tables[0] + assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} + + +def test_schemas_indexes(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + indexes = app_client.get( + "/_schemas/indexes.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(indexes) > 5 + index = indexes[0] + assert set(index.keys()) == { + "partial", + "name", + "table_name", + "unique", + "seq", + "database_name", + "origin", + } + + +def test_schemas_foreign_keys(app_client): + cookie = app_client.actor_cookie({"id": "root"}) + foreign_keys = app_client.get( + "/_schemas/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} + ).json + assert len(foreign_keys) > 5 + foreign_key = foreign_keys[0] + assert set(foreign_key.keys()) == { + "table", + "seq", + "on_update", + "on_delete", + "to", + "rowid", + "id", + "match", + "database_name", + "table_name", + "from", + } From dcdfb2c301341d45b66683e3e3be72f9c7585b2f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Dec 2020 11:48:06 -0800 Subject: [PATCH 0225/1866] Rename _schemas to _internal, closes #1156 --- datasette/app.py | 39 +++++++------------ datasette/cli.py | 4 +- datasette/default_permissions.py | 2 +- datasette/utils/__init__.py | 2 +- .../utils/{schemas.py => internal_db.py} | 20 +++++----- .../{test_schemas.py => test_internal_db.py} | 24 ++++++------ tests/test_plugins.py | 2 +- 7 files changed, 42 insertions(+), 51 deletions(-) rename datasette/utils/{schemas.py => internal_db.py} (91%) rename tests/{test_schemas.py => test_internal_db.py} (63%) diff --git a/datasette/app.py b/datasette/app.py index cc8506e2..f995e79d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,7 +74,7 @@ from .utils.asgi import ( asgi_send_json, asgi_send_redirect, ) -from .utils.schemas import init_schemas, populate_schema_tables +from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, using_pysqlite3, @@ -85,8 +85,6 @@ from .version import __version__ app_root = Path(__file__).parent.parent -MEMORY = object() - Setting = collections.namedtuple("Setting", ("name", "default", "help")) SETTINGS = ( Setting("default_page_size", 100, "Default page size for the table view"), @@ -218,24 +216,17 @@ class Datasette: ] self.inspect_data = inspect_data self.immutables = set(immutables or []) - if not self.files: - self.files = [MEMORY] - elif memory: - self.files = (MEMORY,) + self.files self.databases = collections.OrderedDict() + if memory or not self.files: + self.add_database(":memory:", Database(self, ":memory:", is_memory=True)) # memory_name is a random string so that each Datasette instance gets its own # unique in-memory named database - otherwise unit tests can fail with weird # errors when different instances accidentally share an in-memory database - self.add_database("_schemas", Database(self, memory_name=secrets.token_hex())) - self._schemas_created = False + self.add_database("_internal", Database(self, memory_name=secrets.token_hex())) + self._interna_db_created = False for file in self.files: path = file - is_memory = False - if file is MEMORY: - path = None - is_memory = True - is_mutable = path not in self.immutables - db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) + db = Database(self, path, is_mutable=path not in self.immutables) if db.name in self.databases: raise Exception(f"Multiple files with same stem: {db.name}") self.add_database(db.name, db) @@ -333,14 +324,14 @@ class Datasette: self.client = DatasetteClient(self) async def refresh_schemas(self): - schema_db = self.databases["_schemas"] - if not self._schemas_created: - await init_schemas(schema_db) - self._schemas_created = True + internal_db = self.databases["_internal"] + if not self._interna_db_created: + await init_internal_db(internal_db) + self._interna_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] - for row in await schema_db.execute( + for row in await internal_db.execute( "select database_name, schema_version from databases" ) } @@ -349,7 +340,7 @@ class Datasette: # Compare schema versions to see if we should skip it if schema_version == current_schema_versions.get(database_name): continue - await schema_db.execute_write( + await internal_db.execute_write( """ INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) VALUES (?, ?, ?, ?) @@ -357,7 +348,7 @@ class Datasette: [database_name, db.path, db.is_memory, schema_version], block=True, ) - await populate_schema_tables(schema_db, db) + await populate_schema_tables(internal_db, db) @property def urls(self): @@ -376,7 +367,7 @@ class Datasette: def get_database(self, name=None): if name is None: # Return first no-_schemas database - name = [key for key in self.databases.keys() if key != "_schemas"][0] + name = [key for key in self.databases.keys() if key != "_internal"][0] return self.databases[name] def add_database(self, name, db): @@ -625,7 +616,7 @@ class Datasette: "hash": d.hash, } for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) - if name != "_schemas" + if name != "_internal" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 50367fb3..c342a35a 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -134,8 +134,8 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - if name == "_schemas": - # Don't include the in-memory _schemas database + if name == "_internal": + # Don't include the in-memory _internal database continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 62cab83a..b58d8d1b 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -13,7 +13,7 @@ def permission_allowed(datasette, actor, action, resource): if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - if resource == "_schemas" and (actor is None or actor.get("id") != "root"): + if resource == "_internal" and (actor is None or actor.get("id") != "root"): return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index ac1d82f7..34ee4630 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1024,7 +1024,7 @@ def find_spatialite(): async def initial_path_for_datasette(datasette): "Return suggested path for opening this Datasette, based on number of DBs and tables" - databases = dict([p for p in datasette.databases.items() if p[0] != "_schemas"]) + databases = dict([p for p in datasette.databases.items() if p[0] != "_internal"]) if len(databases) == 1: db_name = next(iter(databases.keys())) path = datasette.urls.database(db_name) diff --git a/datasette/utils/schemas.py b/datasette/utils/internal_db.py similarity index 91% rename from datasette/utils/schemas.py rename to datasette/utils/internal_db.py index 4612e236..a60fe1fe 100644 --- a/datasette/utils/schemas.py +++ b/datasette/utils/internal_db.py @@ -1,4 +1,4 @@ -async def init_schemas(db): +async def init_internal_db(db): await db.execute_write( """ CREATE TABLE databases ( @@ -73,15 +73,15 @@ async def init_schemas(db): ) -async def populate_schema_tables(schema_db, db): +async def populate_schema_tables(internal_db, db): database_name = db.name - await schema_db.execute_write( + await internal_db.execute_write( "delete from tables where database_name = ?", [database_name], block=True ) tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows for table in tables: table_name = table["name"] - await schema_db.execute_write( + await internal_db.execute_write( """ insert into tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) @@ -90,7 +90,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the columns - await schema_db.execute_write( + await internal_db.execute_write( "delete from columns where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -101,7 +101,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **column._asdict(), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden @@ -113,7 +113,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the foreign_keys - await schema_db.execute_write( + await internal_db.execute_write( "delete from foreign_keys where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -126,7 +126,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **dict(foreign_key), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match @@ -138,7 +138,7 @@ async def populate_schema_tables(schema_db, db): block=True, ) # And the indexes - await schema_db.execute_write( + await internal_db.execute_write( "delete from indexes where database_name = ? and table_name = ?", [database_name, table_name], block=True, @@ -149,7 +149,7 @@ async def populate_schema_tables(schema_db, db): **{"database_name": database_name, "table_name": table_name}, **dict(index), } - await schema_db.execute_write( + await internal_db.execute_write( """ insert into indexes ( database_name, table_name, seq, name, "unique", origin, partial diff --git a/tests/test_schemas.py b/tests/test_internal_db.py similarity index 63% rename from tests/test_schemas.py rename to tests/test_internal_db.py index 87656784..9349fa3c 100644 --- a/tests/test_schemas.py +++ b/tests/test_internal_db.py @@ -2,36 +2,36 @@ from .fixtures import app_client import pytest -def test_schemas_only_available_to_root(app_client): +def test_internal_only_available_to_root(app_client): cookie = app_client.actor_cookie({"id": "root"}) - assert app_client.get("/_schemas").status == 403 - assert app_client.get("/_schemas", cookies={"ds_actor": cookie}).status == 200 + assert app_client.get("/_internal").status == 403 + assert app_client.get("/_internal", cookies={"ds_actor": cookie}).status == 200 -def test_schemas_databases(app_client): +def test_internal_databases(app_client): cookie = app_client.actor_cookie({"id": "root"}) databases = app_client.get( - "/_schemas/databases.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/databases.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(databases) == 2 - assert databases[0]["database_name"] == "_schemas" + assert databases[0]["database_name"] == "_internal" assert databases[1]["database_name"] == "fixtures" -def test_schemas_tables(app_client): +def test_internal_tables(app_client): cookie = app_client.actor_cookie({"id": "root"}) tables = app_client.get( - "/_schemas/tables.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/tables.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(tables) > 5 table = tables[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} -def test_schemas_indexes(app_client): +def test_internal_indexes(app_client): cookie = app_client.actor_cookie({"id": "root"}) indexes = app_client.get( - "/_schemas/indexes.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/indexes.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(indexes) > 5 index = indexes[0] @@ -46,10 +46,10 @@ def test_schemas_indexes(app_client): } -def test_schemas_foreign_keys(app_client): +def test_internal_foreign_keys(app_client): cookie = app_client.actor_cookie({"id": "root"}) foreign_keys = app_client.get( - "/_schemas/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} + "/_internal/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} ).json assert len(foreign_keys) > 5 foreign_key = foreign_keys[0] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 61e7d4b5..8063460b 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -293,7 +293,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): def test_hook_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "_schemas, fixtures" == response.headers["x-databases"] + assert "_internal, fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): From 810853c5f2fa560c6d303331c037f6443c145930 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 21 Dec 2020 13:49:14 -0800 Subject: [PATCH 0226/1866] Use time.perf_counter() instead of time.time(), closes #1157 --- datasette/tracer.py | 8 ++++---- datasette/utils/__init__.py | 4 ++-- datasette/views/base.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 8f666767..772f0405 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -37,9 +37,9 @@ def trace(type, **kwargs): if tracer is None: yield return - start = time.time() + start = time.perf_counter() yield - end = time.time() + end = time.perf_counter() trace_info = { "type": type, "start": start, @@ -74,7 +74,7 @@ class AsgiTracer: if b"_trace=1" not in scope.get("query_string", b"").split(b"&"): await self.app(scope, receive, send) return - trace_start = time.time() + trace_start = time.perf_counter() traces = [] accumulated_body = b"" @@ -109,7 +109,7 @@ class AsgiTracer: # We have all the body - modify it and send the result # TODO: What to do about Content-Type or other cases? trace_info = { - "request_duration_ms": 1000 * (time.time() - trace_start), + "request_duration_ms": 1000 * (time.perf_counter() - trace_start), "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces), "num_traces": len(traces), "traces": traces, diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 34ee4630..0d45e11a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -138,7 +138,7 @@ class CustomJSONEncoder(json.JSONEncoder): @contextmanager def sqlite_timelimit(conn, ms): - deadline = time.time() + (ms / 1000) + deadline = time.perf_counter() + (ms / 1000) # n is the number of SQLite virtual machine instructions that will be # executed between each check. It's hard to know what to pick here. # After some experimentation, I've decided to go with 1000 by default and @@ -148,7 +148,7 @@ def sqlite_timelimit(conn, ms): n = 1 def handler(): - if time.time() >= deadline: + if time.perf_counter() >= deadline: return 1 conn.set_progress_handler(handler, n) diff --git a/datasette/views/base.py b/datasette/views/base.py index 73bf9459..8a64f88e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -425,7 +425,7 @@ class DataView(BaseView): kwargs["default_labels"] = True extra_template_data = {} - start = time.time() + start = time.perf_counter() status_code = 200 templates = [] try: @@ -457,7 +457,7 @@ class DataView(BaseView): except DatasetteError: raise - end = time.time() + end = time.perf_counter() data["query_ms"] = (end - start) * 1000 for key in ("source", "source_url", "license", "license_url"): value = self.ds.metadata(key) From bc1f1e1ce8562872b7532a167873193e787cef20 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 11:04:29 -0800 Subject: [PATCH 0227/1866] Compound primary key for foreign_keys table in _internal --- datasette/utils/internal_db.py | 3 ++- tests/test_internal_db.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index a60fe1fe..959f422e 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -66,7 +66,8 @@ async def init_internal_db(db): "to" TEXT, "on_update" TEXT, "on_delete" TEXT, - "match" TEXT + "match" TEXT, + PRIMARY KEY (database_name, table_name, id, seq) ) """, block=True, diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 9349fa3c..755ddae5 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -59,7 +59,6 @@ def test_internal_foreign_keys(app_client): "on_update", "on_delete", "to", - "rowid", "id", "match", "database_name", From 270de6527bc2afb8c5996c400099321c320ded31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 11:48:54 -0800 Subject: [PATCH 0228/1866] Foreign keys for _internal database Refs #1099 - Datasette now uses compound foreign keys internally, so it would be great to link them correctly. --- datasette/utils/internal_db.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 959f422e..5cd32381 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -17,7 +17,8 @@ async def init_internal_db(db): "table_name" TEXT, "rootpage" INTEGER, "sql" TEXT, - PRIMARY KEY (database_name, table_name) + PRIMARY KEY (database_name, table_name), + FOREIGN KEY (database_name) REFERENCES databases(database_name) ) """, block=True, @@ -34,7 +35,9 @@ async def init_internal_db(db): "default_value" TEXT, -- renamed from dflt_value "is_pk" INTEGER, -- renamed from pk "hidden" INTEGER, - PRIMARY KEY (database_name, table_name, name) + PRIMARY KEY (database_name, table_name, name), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, @@ -49,7 +52,9 @@ async def init_internal_db(db): "unique" INTEGER, "origin" TEXT, "partial" INTEGER, - PRIMARY KEY (database_name, table_name, name) + PRIMARY KEY (database_name, table_name, name), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, @@ -67,7 +72,9 @@ async def init_internal_db(db): "on_update" TEXT, "on_delete" TEXT, "match" TEXT, - PRIMARY KEY (database_name, table_name, id, seq) + PRIMARY KEY (database_name, table_name, id, seq), + FOREIGN KEY (database_name) REFERENCES databases(database_name), + FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) """, block=True, From 8919f99c2f7f245aca7f94bd53d5ac9d04aa42b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 12:04:18 -0800 Subject: [PATCH 0229/1866] Improved .add_database() method design Closes #1155 - _internal now has a sensible name Closes #509 - Support opening multiple databases with the same stem --- datasette/app.py | 34 +++++++++++++++++--------- datasette/database.py | 42 +++++++++++++++++--------------- docs/internals.rst | 29 ++++++++++++++-------- tests/test_cli.py | 15 ++++++++++++ tests/test_internals_database.py | 12 ++++----- 5 files changed, 86 insertions(+), 46 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f995e79d..ad3ba07e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -218,18 +218,18 @@ class Datasette: self.immutables = set(immutables or []) self.databases = collections.OrderedDict() if memory or not self.files: - self.add_database(":memory:", Database(self, ":memory:", is_memory=True)) + self.add_database(Database(self, is_memory=True), name=":memory:") # memory_name is a random string so that each Datasette instance gets its own # unique in-memory named database - otherwise unit tests can fail with weird # errors when different instances accidentally share an in-memory database - self.add_database("_internal", Database(self, memory_name=secrets.token_hex())) - self._interna_db_created = False + self.add_database( + Database(self, memory_name=secrets.token_hex()), name="_internal" + ) + self.internal_db_created = False for file in self.files: - path = file - db = Database(self, path, is_mutable=path not in self.immutables) - if db.name in self.databases: - raise Exception(f"Multiple files with same stem: {db.name}") - self.add_database(db.name, db) + self.add_database( + Database(self, file, is_mutable=file not in self.immutables) + ) self.cache_headers = cache_headers self.cors = cors metadata_files = [] @@ -325,9 +325,9 @@ class Datasette: async def refresh_schemas(self): internal_db = self.databases["_internal"] - if not self._interna_db_created: + if not self.internal_db_created: await init_internal_db(internal_db) - self._interna_db_created = True + self.internal_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] @@ -370,8 +370,20 @@ class Datasette: name = [key for key in self.databases.keys() if key != "_internal"][0] return self.databases[name] - def add_database(self, name, db): + def add_database(self, db, name=None): + if name is None: + # Pick a unique name for this database + suggestion = db.suggest_name() + name = suggestion + else: + suggestion = name + i = 2 + while name in self.databases: + name = "{}_{}".format(suggestion, i) + i += 1 + db.name = name self.databases[name] = db + return db def remove_database(self, name): self.databases.pop(name) diff --git a/datasette/database.py b/datasette/database.py index a977b362..cda36e6e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -27,30 +27,44 @@ class Database: def __init__( self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None ): + self.name = None self.ds = ds self.path = path self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name if memory_name is not None: - self.path = memory_name self.is_memory = True self.is_mutable = True self.hash = None self.cached_size = None - self.cached_table_counts = None + self._cached_table_counts = None self._write_thread = None self._write_queue = None if not self.is_mutable and not self.is_memory: p = Path(path) self.hash = inspect_hash(p) self.cached_size = p.stat().st_size - # Maybe use self.ds.inspect_data to populate cached_table_counts - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.cached_table_counts = { - key: value["count"] - for key, value in self.ds.inspect_data[self.name]["tables"].items() - } + + @property + def cached_table_counts(self): + if self._cached_table_counts is not None: + return self._cached_table_counts + # Maybe use self.ds.inspect_data to populate cached_table_counts + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self._cached_table_counts = { + key: value["count"] + for key, value in self.ds.inspect_data[self.name]["tables"].items() + } + return self._cached_table_counts + + def suggest_name(self): + if self.path: + return Path(self.path).stem + elif self.memory_name: + return self.memory_name + else: + return "db" def connect(self, write=False): if self.memory_name: @@ -220,7 +234,7 @@ class Database: except (QueryInterrupted, sqlite3.OperationalError, sqlite3.DatabaseError): counts[table] = None if not self.is_mutable: - self.cached_table_counts = counts + self._cached_table_counts = counts return counts @property @@ -229,16 +243,6 @@ class Database: return None return Path(self.path).stat().st_mtime_ns - @property - def name(self): - if self.is_memory: - if self.memory_name: - return ":memory:{}".format(self.memory_name) - else: - return ":memory:" - else: - return Path(self.path).stem - async def table_exists(self, table): results = await self.execute( "select 1 from sqlite_master where type='table' and name=?", params=(table,) diff --git a/docs/internals.rst b/docs/internals.rst index b68a1d8a..05cb8bd7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -245,16 +245,16 @@ Returns the specified database object. Raises a ``KeyError`` if the database doe .. _datasette_add_database: -.add_database(name, db) ------------------------ - -``name`` - string - The unique name to use for this database. Also used in the URL. +.add_database(db, name=None) +---------------------------- ``db`` - datasette.database.Database instance The database to be attached. -The ``datasette.add_database(name, db)`` method lets you add a new database to the current Datasette instance. This database will then be served at URL path that matches the ``name`` parameter, e.g. ``/mynewdb/``. +``name`` - string, optional + The name to be used for this database - this will be used in the URL path, e.g. ``/dbname``. If not specified Datasette will pick one based on the filename or memory name. + +The ``datasette.add_database(db)`` method lets you add a new database to the current Datasette instance. The ``db`` parameter should be an instance of the ``datasette.database.Database`` class. For example: @@ -262,13 +262,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` from datasette.database import Database - datasette.add_database("my-new-database", Database( + datasette.add_database(Database( datasette, path="path/to/my-new-database.db", is_mutable=True )) -This will add a mutable database from the provided file path. +This will add a mutable database and serve it at ``/my-new-database``. To create a shared in-memory database named ``statistics``, use the following: @@ -276,11 +276,20 @@ To create a shared in-memory database named ``statistics``, use the following: from datasette.database import Database - datasette.add_database("statistics", Database( + datasette.add_database(Database( datasette, memory_name="statistics" )) +This database will be served at ``/statistics``. + +``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: + +.. code-block:: python + + db = datasette.add_database(Database(datasette, memory_name="statistics")) + await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) + .. _datasette_remove_database: .remove_database(name) @@ -289,7 +298,7 @@ To create a shared in-memory database named ``statistics``, use the following: ``name`` - string The name of the database to be removed. -This removes a database that has been previously added. ``name=`` is the unique name of that database, also used in the URL for it. +This removes a database that has been previously added. ``name=`` is the unique name of that database, used in its URL path. .. _datasette_sign: diff --git a/tests/test_cli.py b/tests/test_cli.py index 3f6b1840..ff46d76f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -8,6 +8,7 @@ import asyncio from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ +from datasette.utils.sqlite import sqlite3 from click.testing import CliRunner import io import json @@ -240,3 +241,17 @@ def test_serve_create(ensure_eventloop, tmpdir): "hash": None, }.items() <= databases[0].items() assert db_path.exists() + + +def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): + runner = CliRunner() + db_1_path = str(tmpdir / "db.db") + nested = tmpdir / "nested" + nested.mkdir() + db_2_path = str(tmpdir / "nested" / "db.db") + for path in (db_1_path, db_2_path): + sqlite3.connect(path).execute("vacuum") + result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) + assert result.exit_code == 0, result.output + databases = json.loads(result.output) + assert {db["name"] for db in databases} == {"db", "db_2"} diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index dc1af48c..7eff9f7e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -439,7 +439,7 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") sqlite3.connect(path).execute("vacuum") db = Database(app_client.ds, path=path, is_mutable=False) - app_client.ds.add_database("immutable-db", db) + app_client.ds.add_database(db, name="immutable-db") def write_fn(conn): assert False @@ -469,10 +469,10 @@ def test_is_mutable(app_client): @pytest.mark.asyncio async def test_database_memory_name(app_client): ds = app_client.ds - foo1 = Database(ds, memory_name="foo") - foo2 = Database(ds, memory_name="foo") - bar1 = Database(ds, memory_name="bar") - bar2 = Database(ds, memory_name="bar") + foo1 = ds.add_database(Database(ds, memory_name="foo")) + foo2 = ds.add_database(Database(ds, memory_name="foo")) + bar1 = ds.add_database(Database(ds, memory_name="bar")) + bar2 = ds.add_database(Database(ds, memory_name="bar")) for db in (foo1, foo2, bar1, bar2): table_names = await db.table_names() assert table_names == [] @@ -487,7 +487,7 @@ async def test_database_memory_name(app_client): @pytest.mark.asyncio async def test_in_memory_databases_forbid_writes(app_client): ds = app_client.ds - db = Database(ds, memory_name="test") + db = ds.add_database(Database(ds, memory_name="test")) with pytest.raises(sqlite3.OperationalError): await db.execute("create table foo (t text)") assert await db.table_names() == [] From 90eba4c3ca569c57e96bce314e7ac8caf67d884e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Dec 2020 15:55:43 -0800 Subject: [PATCH 0230/1866] Prettier CREATE TABLE SQL for _internal --- datasette/utils/internal_db.py | 109 ++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 5cd32381..e92625d5 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,82 +1,95 @@ +import textwrap + + async def init_internal_db(db): await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE databases ( - "database_name" TEXT PRIMARY KEY, - "path" TEXT, - "is_memory" INTEGER, - "schema_version" INTEGER + database_name TEXT PRIMARY KEY, + path TEXT, + is_memory INTEGER, + schema_version INTEGER ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE tables ( - "database_name" TEXT, - "table_name" TEXT, - "rootpage" INTEGER, - "sql" TEXT, + database_name TEXT, + table_name TEXT, + rootpage INTEGER, + sql TEXT, PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE columns ( - "database_name" TEXT, - "table_name" TEXT, - "cid" INTEGER, - "name" TEXT, - "type" TEXT, + database_name TEXT, + table_name TEXT, + cid INTEGER, + name TEXT, + type TEXT, "notnull" INTEGER, - "default_value" TEXT, -- renamed from dflt_value - "is_pk" INTEGER, -- renamed from pk - "hidden" INTEGER, + default_value TEXT, -- renamed from dflt_value + is_pk INTEGER, -- renamed from pk + hidden INTEGER, PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE indexes ( - "database_name" TEXT, - "table_name" TEXT, - "seq" INTEGER, - "name" TEXT, + database_name TEXT, + table_name TEXT, + seq INTEGER, + name TEXT, "unique" INTEGER, - "origin" TEXT, - "partial" INTEGER, + origin TEXT, + partial INTEGER, PRIMARY KEY (database_name, table_name, name), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) await db.execute_write( - """ + textwrap.dedent( + """ CREATE TABLE foreign_keys ( - "database_name" TEXT, - "table_name" TEXT, - "id" INTEGER, - "seq" INTEGER, + database_name TEXT, + table_name TEXT, + id INTEGER, + seq INTEGER, "table" TEXT, "from" TEXT, "to" TEXT, - "on_update" TEXT, - "on_delete" TEXT, - "match" TEXT, + on_update TEXT, + on_delete TEXT, + match TEXT, PRIMARY KEY (database_name, table_name, id, seq), FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ) - """, + """ + ), block=True, ) @@ -84,14 +97,14 @@ async def init_internal_db(db): async def populate_schema_tables(internal_db, db): database_name = db.name await internal_db.execute_write( - "delete from tables where database_name = ?", [database_name], block=True + "DELETE FROM tables WHERE database_name = ?", [database_name], block=True ) - tables = (await db.execute("select * from sqlite_master where type = 'table'")).rows + tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows for table in tables: table_name = table["name"] await internal_db.execute_write( """ - insert into tables (database_name, table_name, rootpage, sql) + INSERT INTO tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, [database_name, table_name, table["rootpage"], table["sql"]], @@ -99,7 +112,7 @@ async def populate_schema_tables(internal_db, db): ) # And the columns await internal_db.execute_write( - "delete from columns where database_name = ? and table_name = ?", + "DELETE FROM columns WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -111,7 +124,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into columns ( + INSERT INTO columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -122,7 +135,7 @@ async def populate_schema_tables(internal_db, db): ) # And the foreign_keys await internal_db.execute_write( - "delete from foreign_keys where database_name = ? and table_name = ?", + "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -136,7 +149,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into foreign_keys ( + INSERT INTO foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -147,7 +160,7 @@ async def populate_schema_tables(internal_db, db): ) # And the indexes await internal_db.execute_write( - "delete from indexes where database_name = ? and table_name = ?", + "DELETE FROM indexes WHERE database_name = ? and table_name = ?", [database_name, table_name], block=True, ) @@ -159,7 +172,7 @@ async def populate_schema_tables(internal_db, db): } await internal_db.execute_write( """ - insert into indexes ( + INSERT INTO indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial From a882d679626438ba0d809944f06f239bcba8ee96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <6774676+eumiro@users.noreply.github.com> Date: Wed, 23 Dec 2020 18:04:32 +0100 Subject: [PATCH 0231/1866] Modernize code to Python 3.6+ (#1158) * Compact dict and set building * Remove redundant parentheses * Simplify chained conditions * Change method name to lowercase * Use triple double quotes for docstrings Thanks, @eumiro! --- datasette/app.py | 16 +++++++------- datasette/cli.py | 10 ++++----- datasette/facets.py | 4 +--- datasette/filters.py | 6 +++--- datasette/hookspecs.py | 42 ++++++++++++++++++------------------- datasette/inspect.py | 6 +++--- datasette/renderer.py | 2 +- datasette/utils/__init__.py | 20 +++++++++--------- datasette/utils/asgi.py | 18 +++++++--------- datasette/views/base.py | 6 +++--- datasette/views/table.py | 4 ++-- tests/fixtures.py | 2 +- tests/plugins/my_plugin.py | 2 +- tests/test_api.py | 4 ++-- tests/test_auth.py | 4 ++-- tests/test_cli.py | 2 +- tests/test_docs.py | 6 +++--- tests/test_permissions.py | 2 +- tests/test_plugins.py | 2 +- 19 files changed, 76 insertions(+), 82 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ad3ba07e..bd62fd3b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -429,7 +429,7 @@ class Datasette: return m def plugin_config(self, plugin_name, database=None, table=None, fallback=True): - "Return config for plugin, falling back from specified database/table" + """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( "plugins", database=database, table=table, fallback=fallback ) @@ -523,7 +523,7 @@ class Datasette: return [] async def permission_allowed(self, actor, action, resource=None, default=False): - "Check permissions using the permissions_allowed plugin hook" + """Check permissions using the permissions_allowed plugin hook""" result = None for check in pm.hook.permission_allowed( datasette=self, @@ -570,7 +570,7 @@ class Datasette: ) async def expand_foreign_keys(self, database, table, column, values): - "Returns dict mapping (column, value) -> label" + """Returns dict mapping (column, value) -> label""" labeled_fks = {} db = self.databases[database] foreign_keys = await db.foreign_keys_for_table(table) @@ -613,7 +613,7 @@ class Datasette: return url def _register_custom_units(self): - "Register any custom units defined in the metadata.json with Pint" + """Register any custom units defined in the metadata.json with Pint""" for unit in self.metadata("custom_units") or []: ureg.define(unit) @@ -730,7 +730,7 @@ class Datasette: return {"actor": request.actor} def table_metadata(self, database, table): - "Fetch table-specific metadata." + """Fetch table-specific metadata.""" return ( (self.metadata("databases") or {}) .get(database, {}) @@ -739,7 +739,7 @@ class Datasette: ) def _register_renderers(self): - """ Register output renderers which output data in custom formats. """ + """Register output renderers which output data in custom formats.""" # Built-in renderers self.renderers["json"] = (json_renderer, lambda: True) @@ -880,7 +880,7 @@ class Datasette: return output def app(self): - "Returns an ASGI app function that serves the whole of Datasette" + """Returns an ASGI app function that serves the whole of Datasette""" routes = [] for routes_to_add in pm.hook.register_routes(): @@ -1287,7 +1287,7 @@ def permanent_redirect(path): ) -_curly_re = re.compile(r"(\{.*?\})") +_curly_re = re.compile(r"({.*?})") def route_pattern_from_filepath(filepath): diff --git a/datasette/cli.py b/datasette/cli.py index c342a35a..2a84bf30 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -152,7 +152,7 @@ async def inspect_(files, sqlite_extensions): @cli.group() def publish(): - "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API" + """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API""" pass @@ -168,7 +168,7 @@ pm.hook.publish_subcommand(publish=publish) help="Path to directory containing custom plugins", ) def plugins(all, plugins_dir): - "List currently available plugins" + """List currently available plugins""" app = Datasette([], plugins_dir=plugins_dir) click.echo(json.dumps(app._plugins(all=all), indent=4)) @@ -244,7 +244,7 @@ def package( port, **extra_metadata, ): - "Package specified SQLite files into a new datasette Docker container" + """Package specified SQLite files into a new datasette Docker container""" if not shutil.which("docker"): click.secho( ' The package command requires "docker" to be installed and configured ', @@ -284,7 +284,7 @@ def package( "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" ) def install(packages, upgrade): - "Install Python packages - e.g. Datasette plugins - into the same environment as Datasette" + """Install Python packages - e.g. Datasette plugins - into the same environment as Datasette""" args = ["pip", "install"] if upgrade: args += ["--upgrade"] @@ -297,7 +297,7 @@ def install(packages, upgrade): @click.argument("packages", nargs=-1, required=True) @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") def uninstall(packages, yes): - "Uninstall Python packages (e.g. plugins) from the Datasette environment" + """Uninstall Python packages (e.g. plugins) from the Datasette environment""" sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) run_module("pip", run_name="__main__") diff --git a/datasette/facets.py b/datasette/facets.py index 8ad5a423..207d819d 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -162,10 +162,8 @@ class ColumnFacet(Facet): ) num_distinct_values = len(distinct_values) if ( - num_distinct_values - and num_distinct_values > 1 + 1 < num_distinct_values < row_count and num_distinct_values <= facet_size - and num_distinct_values < row_count # And at least one has n > 1 and any(r["n"] > 1 for r in distinct_values) ): diff --git a/datasette/filters.py b/datasette/filters.py index edf2de99..152a26b4 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -208,7 +208,7 @@ class Filters: self.ureg = ureg def lookups(self): - "Yields (lookup, display, no_argument) pairs" + """Yields (lookup, display, no_argument) pairs""" for filter in self._filters: yield filter.key, filter.display, filter.no_argument @@ -233,7 +233,7 @@ class Filters: return f"where {s}" def selections(self): - "Yields (column, lookup, value) tuples" + """Yields (column, lookup, value) tuples""" for key, value in self.pairs: if "__" in key: column, lookup = key.rsplit("__", 1) @@ -246,7 +246,7 @@ class Filters: return bool(self.pairs) def convert_unit(self, column, value): - "If the user has provided a unit in the query, convert it into the column unit, if present." + """If the user has provided a unit in the query, convert it into the column unit, if present.""" if column not in self.units: return value diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a305ca6a..13a10680 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -7,108 +7,108 @@ hookimpl = HookimplMarker("datasette") @hookspec def startup(datasette): - "Fires directly after Datasette first starts running" + """Fires directly after Datasette first starts running""" @hookspec def asgi_wrapper(datasette): - "Returns an ASGI middleware callable to wrap our ASGI application with" + """Returns an ASGI middleware callable to wrap our ASGI application with""" @hookspec def prepare_connection(conn, database, datasette): - "Modify SQLite connection in some way e.g. register custom SQL functions" + """Modify SQLite connection in some way e.g. register custom SQL functions""" @hookspec def prepare_jinja2_environment(env): - "Modify Jinja2 template environment e.g. register custom template tags" + """Modify Jinja2 template environment e.g. register custom template tags""" @hookspec def extra_css_urls(template, database, table, columns, view_name, request, datasette): - "Extra CSS URLs added by this plugin" + """Extra CSS URLs added by this plugin""" @hookspec def extra_js_urls(template, database, table, columns, view_name, request, datasette): - "Extra JavaScript URLs added by this plugin" + """Extra JavaScript URLs added by this plugin""" @hookspec def extra_body_script( template, database, table, columns, view_name, request, datasette ): - "Extra JavaScript code to be included in + {% endfor %} {% block extra_head %}{% endblock %} diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index d37bb729..a7236873 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -5,6 +5,8 @@ Custom pages and templates Datasette provides a number of ways of customizing the way data is displayed. +.. _customization_css_and_javascript: + Custom CSS and JavaScript ------------------------- @@ -25,7 +27,12 @@ Your ``metadata.json`` file can include links that look like this: ] } -The extra CSS and JavaScript files will be linked in the ```` of every page. +The extra CSS and JavaScript files will be linked in the ```` of every page: + +.. code-block:: html + + + You can also specify a SRI (subresource integrity hash) for these assets: @@ -46,9 +53,39 @@ You can also specify a SRI (subresource integrity hash) for these assets: ] } +This will produce: + +.. code-block:: html + + + + Modern browsers will only execute the stylesheet or JavaScript if the SRI hash matches the content served. You can generate hashes using `www.srihash.org `_ +Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: + +.. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } + +Will produce this HTML: + +.. code-block:: html + + + CSS classes on the ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 72b09367..d465307b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -182,7 +182,7 @@ This can be a list of URLs: @hookimpl def extra_css_urls(): return [ - 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css' + "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css" ] Or a list of dictionaries defining both a URL and an @@ -190,21 +190,17 @@ Or a list of dictionaries defining both a URL and an .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_css_urls(): return [{ - 'url': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css', - 'sri': 'sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4', + "url": "https://stackpath.bootstrapcdn.com/bootstrap/4.1.0/css/bootstrap.min.css", + "sri": "sha384-9gVQ4dYFwwWSjIDZnLEWnxCjeSWFphJiwGPXr1jddIhOegiu1FwO5qRGvFXOdJZ4", }] This function can also return an awaitable function, useful if it needs to run any async code: .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_css_urls(datasette): async def inner(): @@ -233,8 +229,8 @@ return a list of URLs, a list of dictionaries or an awaitable function that retu @hookimpl def extra_js_urls(): return [{ - 'url': 'https://code.jquery.com/jquery-3.3.1.slim.min.js', - 'sri': 'sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo', + "url": "https://code.jquery.com/jquery-3.3.1.slim.min.js", + "sri": "sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo", }] You can also return URLs to files from your plugin's ``static/`` directory, if @@ -242,12 +238,21 @@ you have one: .. code-block:: python - from datasette import hookimpl - @hookimpl def extra_js_urls(): return [ - '/-/static-plugins/your-plugin/app.js' + "/-/static-plugins/your-plugin/app.js" + ] + +If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. + +.. code-block:: python + + @hookimpl + def extra_js_urls(): + return [{ + "url": "/-/static-plugins/your-plugin/app.js", + "module": True ] Examples: `datasette-cluster-map `_, `datasette-vega `_ diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 2e653e2b..1c86b4bc 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -61,6 +61,7 @@ def extra_js_urls(): "sri": "SRIHASH", }, "https://plugin-example.datasette.io/plugin1.js", + {"url": "https://plugin-example.datasette.io/plugin.module.js", "module": True}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 212de2b5..648e7abd 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -118,16 +118,19 @@ def test_hook_extra_css_urls(app_client, path, expected_decoded_object): def test_hook_extra_js_urls(app_client): response = app_client.get("/") scripts = Soup(response.body, "html.parser").findAll("script") - assert [ - s - for s in scripts - if s.attrs - == { + script_attrs = [s.attrs for s in scripts] + for attrs in [ + { "integrity": "SRIHASH", "crossorigin": "anonymous", "src": "https://plugin-example.datasette.io/jquery.js", - } - ] + }, + { + "src": "https://plugin-example.datasette.io/plugin.module.js", + "type": "module", + }, + ]: + assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs) def test_plugins_with_duplicate_js_urls(app_client): From c38c42948cbfddd587729413fd6082ba352eaece Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Jan 2021 18:14:33 -0800 Subject: [PATCH 0253/1866] extra_body_script module support, closes #1187 --- datasette/app.py | 8 +++++++- datasette/templates/base.html | 2 +- docs/plugin_hooks.rst | 25 ++++++++++++++++++++----- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 2 +- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f8549fac..cfce8e0b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -781,7 +781,13 @@ class Datasette: datasette=self, ): extra_script = await await_me_maybe(extra_script) - body_scripts.append(Markup(extra_script)) + if isinstance(extra_script, dict): + script = extra_script["script"] + module = bool(extra_script.get("module")) + else: + script = extra_script + module = False + body_scripts.append({"script": Markup(script), "module": module}) extra_template_vars = {} # pylint: disable=no-member diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 3f3d4507..e61edc4f 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -62,7 +62,7 @@ {% include "_close_open_menus.html" %} {% for body_script in body_scripts %} - + {{ body_script.script }} {% endfor %} {% if select_templates %}{% endif %} diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d465307b..0206daaa 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -168,7 +168,7 @@ Examples: `datasette-search-all extra_css_urls(template, database, table, columns, view_name, request, datasette) --------------------------------------------------------------------------------- -Same arguments as :ref:`extra_template_vars(...) ` +This takes the same arguments as :ref:`extra_template_vars(...) ` Return a list of extra CSS URLs that should be included on the page. These can take advantage of the CSS class hooks described in :ref:`customization`. @@ -217,7 +217,7 @@ Examples: `datasette-cluster-map ` +This takes the same arguments as :ref:`extra_template_vars(...) ` This works in the same way as ``extra_css_urls()`` but for JavaScript. You can return a list of URLs, a list of dictionaries or an awaitable function that returns those things: @@ -264,15 +264,30 @@ extra_body_script(template, database, table, columns, view_name, request, datase Extra JavaScript to be added to a ```` element: + +.. code-block:: python + + @hookimpl + def extra_body_script(): + return { + "module": True, + "script": "console.log('Your JavaScript goes here...')" + } + +This will add the following to the end of your page: + +.. code-block:: html + + Example: `datasette-cluster-map `_ diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 1c86b4bc..8d192d28 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -70,7 +70,7 @@ def extra_body_script( template, database, table, view_name, columns, request, datasette ): async def inner(): - return "var extra_body_script = {};".format( + script = "var extra_body_script = {};".format( json.dumps( { "template": template, @@ -90,6 +90,7 @@ def extra_body_script( } ) ) + return {"script": script, "module": True} return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 648e7abd..715c7c17 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -288,7 +288,7 @@ def test_plugin_config_file(app_client): ], ) def test_hook_extra_body_script(app_client, path, expected_extra_body_script): - r = re.compile(r"") + r = re.compile(r"") json_data = r.search(app_client.get(path).text).group(1) actual_data = json.loads(json_data) assert expected_extra_body_script == actual_data From 7e3cfd9cf7aeddf153d907bc3ee08ae0cd489370 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:27:45 -0800 Subject: [PATCH 0254/1866] Clarify the name of plugin used in /-/static-plugins/ --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 0206daaa..23e57278 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -244,6 +244,8 @@ you have one: "/-/static-plugins/your-plugin/app.js" ] +Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. + If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. .. code-block:: python From 57f4d7b82f9c74298c67c5640207241925b70c02 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:47:30 -0800 Subject: [PATCH 0255/1866] Release 0.54a0 Refs #1091, #1145, #1151, #1156, #1157, #1158, #1166, #1170, #1178, #1182, #1184, #1185, #1186, #1187 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index a5edecfa..b19423a9 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.53" +__version__ = "0.54a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 40b9c5a3..ac2ac8c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_54_a0: + +0.54a0 (2020-12-19) +------------------- + +**Alpha release**. Release notes in progress. + +- Improved support for named in-memory databases. (`#1151 `__) +- New ``_internal`` in-memory database tracking attached databases, tables and columns. (`#1150 `__) +- Support for JavaScript modules. (`#1186 `__, `#1187 `__) + .. _v0_53: 0.53 (2020-12-10) From 5378f023529107ff7edbd6ee4ecab6ac170a83db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Jan 2021 12:50:12 -0800 Subject: [PATCH 0256/1866] Better tool for extracting issue numbers --- docs/contributing.rst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 24d5c8f0..3a4b2caa 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -169,17 +169,7 @@ To release a new version, first create a commit that updates the version number Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. -You can generate the list of issue references for a specific release by pasting the following into the browser devtools while looking at the :ref:`changelog` page (replace ``v0-44`` with the most recent version): - -.. code-block:: javascript - - [ - ...new Set( - Array.from( - document.getElementById("v0-44").querySelectorAll("a[href*=issues]") - ).map((a) => "#" + a.href.split("/issues/")[1]) - ), - ].sort().join(", "); +You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text `__ tool. To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. From 25c2933667680db045851b2cedcf4666d737d352 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Jan 2021 16:46:16 -0800 Subject: [PATCH 0257/1866] publish heroku now uses python-3.8.7 --- datasette/publish/heroku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index c772b476..c0c70e12 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -173,7 +173,7 @@ def temporary_heroku_directory( if metadata_content: open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.8.6") + open("runtime.txt", "w").write("python-3.8.7") if branch: install = [ From f78e956eca1f363e3a3f93c69fd9fc31bed14629 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 12:38:29 -0800 Subject: [PATCH 0258/1866] Plugin testing documentation on using pytest-httpx Closes #1198 --- docs/testing_plugins.rst | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index bacfd57b..4261f639 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -118,3 +118,74 @@ If you want to create that test database repeatedly for every individual test fu @pytest.fixture def datasette(tmp_path_factory): # This fixture will be executed repeatedly for every test + +.. _testing_plugins_pytest_httpx: + +Testing outbound HTTP calls with pytest-httpx +--------------------------------------------- + +If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests. + +The `pytest-httpx `__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally. + +To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture. + +As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content: + +.. code-block:: python + + from datasette import hookimpl + from datasette.utils.asgi import Response + import httpx + + + @hookimpl + def register_routes(): + return [ + (r"^/-/fetch-url$", fetch_url), + ] + + + async def fetch_url(datasette, request): + if request.method == "GET": + return Response.html( + """ +
    + + + """.format( + request.scope["csrftoken"]() + ) + ) + vars = await request.post_vars() + url = vars["url"] + return Response.text(httpx.get(url).text) + +Here's a test for that plugin that mocks the HTTPX outbound request: + +.. code-block:: python + + from datasette.app import Datasette + import pytest + + + @pytest.fixture + def non_mocked_hosts(): + # This ensures httpx-mock will not affect Datasette's own + # httpx calls made in the tests by datasette.client: + return ["localhost"] + + + async def test_outbound_http_call(httpx_mock): + httpx_mock.add_response( + url='https://www.example.com/', + data='Hello world', + ) + datasette = Datasette([], memory=True) + response = await datasette.client.post("/-/fetch-url", data={ + "url": "https://www.example.com/" + }) + asert response.text == "Hello world" + + outbound_request = httpx_mock.get_request() + assert outbound_request.url == "https://www.example.com/" From b6a7b58fa01af0cd5a5e94bd17d686d283a46819 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 16:08:29 -0800 Subject: [PATCH 0259/1866] Initial docs for _internal database, closes #1154 --- docs/internals.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index f7b0cc0b..4a2c0a8e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -747,3 +747,19 @@ If your plugin implements a ``
    `` anywhere you will need to i .. code-block:: html + +.. _internals_internal: + +The _internal database +====================== + +.. warning:: + This API should be considered unstable - the structure of these tables may change prior to the release of Datasette 1.0. + +Datasette maintains an in-memory SQLite database with details of the the databases, tables and columns for all of the attached databases. + +By default all actors are denied access to the ``view-database`` permission for the ``_internal`` database, so the database is not visible to anyone unless they :ref:`sign in as root `. + +Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. + +You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. \ No newline at end of file From ffff3a4c5398a9f40b61d59736f386444da19289 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 17:41:46 -0800 Subject: [PATCH 0260/1866] Easier way to run Prettier locally (#1203) Thanks, Ben Pickles - refs #1167 --- .github/workflows/prettier.yml | 2 +- package.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index d846cca7..9dfe7ee0 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -19,4 +19,4 @@ jobs: run: npm ci - name: Run prettier run: |- - npx --no-install prettier --check 'datasette/static/*[!.min].js' + npm run prettier -- --check diff --git a/package.json b/package.json index 67452d2f..5c6dfe61 100644 --- a/package.json +++ b/package.json @@ -3,5 +3,9 @@ "private": true, "devDependencies": { "prettier": "^2.2.1" + }, + "scripts": { + "fix": "npm run prettier -- --write", + "prettier": "prettier 'datasette/static/*[!.min].js'" } } From f3a155531807c586e62b8ff0e97b96a76e949c8d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 17:58:15 -0800 Subject: [PATCH 0261/1866] Contributing docs for Black and Prettier, closes #1167 Refs #1203 --- docs/contributing.rst | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3a4b2caa..2cf641fd 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -97,6 +97,58 @@ You can tell Datasette to open an interactive ``pdb`` debugger session if an err datasette --pdb fixtures.db +.. _contributing_formatting: + +Code formatting +--------------- + +Datasette uses opinionated code formatters: `Black `__ for Python and `Prettier `__ for JavaScript. + +These formatters are enforced by Datasette's continuous integration: if a commit includes Python or JavaScript code that does not match the style enforced by those tools, the tests will fail. + +When developing locally, you can verify and correct the formatting of your code using these tools. + +.. _contributing_formatting_black: + +Running Black +~~~~~~~~~~~~~ + +Black will be installed when you run ``pip install -e '.[test]'``. To test that your code complies with Black, run the following in your root ``datasette`` repository checkout:: + + $ black . --check + All done! ✨ 🍰 ✨ + 95 files would be left unchanged. + +If any of your code does not conform to Black you can run this to automatically fix those problems:: + + $ black . + reformatted ../datasette/setup.py + All done! ✨ 🍰 ✨ + 1 file reformatted, 94 files left unchanged. + +.. _contributing_formatting_prettier: + +Prettier +~~~~~~~~ + +To install Prettier, `install Node.js `__ and then run the following in the root of your ``datasette`` repository checkout:: + + $ npm install + +This will install Prettier in a ``node_modules`` directory. You can then check that your code matches the coding style like so:: + + $ npm run prettier -- --check + > prettier + > prettier 'datasette/static/*[!.min].js' "--check" + + Checking formatting... + [warn] datasette/static/plugins.js + [warn] Code style issues found in the above file(s). Forgot to run Prettier? + +You can fix any problems by running:: + + $ npm run fix + .. _contributing_documentation: Editing and building the documentation From 07e163561592c743e4117f72102fcd350a600909 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 19:10:10 -0800 Subject: [PATCH 0262/1866] All ?_ parameters now copied to hidden form fields, closes #1194 --- datasette/views/table.py | 17 +++++------------ tests/test_html.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index cc8ef9f1..0a3504b3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -812,19 +812,12 @@ class TableView(RowTableShared): .get(table, {}) ) self.ds.update_with_inherited_metadata(metadata) + form_hidden_args = [] - # Add currently selected facets - for arg in special_args: - if arg == "_facet" or arg.startswith("_facet_"): - form_hidden_args.extend( - (arg, item) for item in request.args.getlist(arg) - ) - for arg in ("_fts_table", "_fts_pk"): - if arg in special_args: - form_hidden_args.append((arg, special_args[arg])) - if request.args.get("_where"): - for where_text in request.args.getlist("_where"): - form_hidden_args.append(("_where", where_text)) + for key in request.args: + if key.startswith("_"): + for value in request.args.getlist(key): + form_hidden_args.append((key, value)) # if no sort specified AND table has a single primary key, # set sort to that so arrow is displayed diff --git a/tests/test_html.py b/tests/test_html.py index c7dd9d97..08d17ca7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1250,6 +1250,28 @@ def test_extra_where_clauses(app_client): ] +@pytest.mark.parametrize( + "path,expected_hidden", + [ + ("/fixtures/facetable?_size=10", [("_size", "10")]), + ( + "/fixtures/facetable?_size=10&_ignore=1&_ignore=2", + [ + ("_size", "10"), + ("_ignore", "1"), + ("_ignore", "2"), + ], + ), + ], +) +def test_other_hidden_form_fields(app_client, path, expected_hidden): + response = app_client.get(path) + soup = Soup(response.body, "html.parser") + inputs = soup.find("form").findAll("input") + hiddens = [i for i in inputs if i["type"] == "hidden"] + assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == expected_hidden + + def test_binary_data_display_in_table(app_client): response = app_client.get("/fixtures/binary_data") assert response.status == 200 From a5ede3cdd455e2bb1a1fb2f4e1b5a9855caf5179 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 Jan 2021 21:13:05 -0800 Subject: [PATCH 0263/1866] Fixed bug loading database called 'test-database (1).sqlite' Closes #1181. Also now ensures that database URLs have special characters URL-quoted. --- datasette/url_builder.py | 6 ++++-- datasette/views/base.py | 3 ++- docs/changelog.rst | 10 ++++++---- tests/test_api.py | 14 +++++++------- tests/test_cli.py | 23 +++++++++++++++++++++++ tests/test_html.py | 6 +++--- tests/test_internals_urls.py | 20 ++++++++++---------- 7 files changed, 55 insertions(+), 27 deletions(-) diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 3034b664..2bcda869 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -30,9 +30,11 @@ class Urls: def database(self, database, format=None): db = self.ds.databases[database] if self.ds.setting("hash_urls") and db.hash: - path = self.path(f"{database}-{db.hash[:HASH_LENGTH]}", format=format) + path = self.path( + f"{urllib.parse.quote(database)}-{db.hash[:HASH_LENGTH]}", format=format + ) else: - path = self.path(database, format=format) + path = self.path(urllib.parse.quote(database), format=format) return path def table(self, database, table, format=None): diff --git a/datasette/views/base.py b/datasette/views/base.py index a21b9298..ba0f7d4c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -181,6 +181,7 @@ class DataView(BaseView): async def resolve_db_name(self, request, db_name, **kwargs): hash = None name = None + db_name = urllib.parse.unquote_plus(db_name) if db_name not in self.ds.databases and "-" in db_name: # No matching DB found, maybe it's a name-hash? name_bit, hash_bit = db_name.rsplit("-", 1) @@ -191,7 +192,7 @@ class DataView(BaseView): hash = hash_bit else: name = db_name - name = urllib.parse.unquote_plus(name) + try: db = self.ds.databases[name] except KeyError: diff --git a/docs/changelog.rst b/docs/changelog.rst index ac2ac8c9..abc2f4f9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,12 +4,14 @@ Changelog ========= -.. _v0_54_a0: +.. _v0_54: + +0.54 (2021-01-24) +----------------- + + -0.54a0 (2020-12-19) -------------------- -**Alpha release**. Release notes in progress. - Improved support for named in-memory databases. (`#1151 `__) - New ``_internal`` in-memory database tracking attached databases, tables and columns. (`#1150 `__) diff --git a/tests/test_api.py b/tests/test_api.py index 3b4f3437..0d1bddd3 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -609,17 +609,17 @@ def test_no_files_uses_memory_database(app_client_no_files): assert response.status == 200 assert { ":memory:": { + "name": ":memory:", "hash": None, "color": "f7935d", + "path": "/%3Amemory%3A", + "tables_and_views_truncated": [], + "tables_and_views_more": False, + "tables_count": 0, + "table_rows_sum": 0, + "show_table_row_counts": False, "hidden_table_rows_sum": 0, "hidden_tables_count": 0, - "name": ":memory:", - "show_table_row_counts": False, - "path": "/:memory:", - "table_rows_sum": 0, - "tables_count": 0, - "tables_and_views_more": False, - "tables_and_views_truncated": [], "views_count": 0, "private": False, } diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d806bff..c42c22ea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,6 +17,7 @@ import pytest import sys import textwrap from unittest import mock +import urllib @pytest.fixture @@ -255,3 +256,25 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert result.exit_code == 0, result.output databases = json.loads(result.output) assert {db["name"] for db in databases} == {"db", "db_2"} + + +@pytest.mark.parametrize( + "filename", ["test-database (1).sqlite", "database (1).sqlite"] +) +def test_weird_database_names(ensure_eventloop, tmpdir, filename): + # https://github.com/simonw/datasette/issues/1181 + runner = CliRunner() + db_path = str(tmpdir / filename) + sqlite3.connect(db_path).execute("vacuum") + result1 = runner.invoke(cli, [db_path, "--get", "/"]) + assert result1.exit_code == 0, result1.output + filename_no_stem = filename.rsplit(".", 1)[0] + expected_link = '{}'.format( + urllib.parse.quote(filename_no_stem), filename_no_stem + ) + assert expected_link in result1.output + # Now try hitting that database page + result2 = runner.invoke( + cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))] + ) + assert result2.exit_code == 0, result2.output diff --git a/tests/test_html.py b/tests/test_html.py index 08d17ca7..6c33fba7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -30,7 +30,7 @@ def test_homepage(app_client_two_attached_databases): # Should be two attached databases assert [ {"href": "/fixtures", "text": "fixtures"}, - {"href": "/extra database", "text": "extra database"}, + {"href": r"/extra%20database", "text": "extra database"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables h2 = soup.select("h2")[1] @@ -44,8 +44,8 @@ def test_homepage(app_client_two_attached_databases): {"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a") ] assert [ - {"href": "/extra database/searchable", "text": "searchable"}, - {"href": "/extra database/searchable_view", "text": "searchable_view"}, + {"href": r"/extra%20database/searchable", "text": "searchable"}, + {"href": r"/extra%20database/searchable_view", "text": "searchable_view"}, ] == table_links diff --git a/tests/test_internals_urls.py b/tests/test_internals_urls.py index fd05c1b6..e6f405b3 100644 --- a/tests/test_internals_urls.py +++ b/tests/test_internals_urls.py @@ -103,9 +103,9 @@ def test_logout(ds, base_url, expected): @pytest.mark.parametrize( "base_url,format,expected", [ - ("/", None, "/:memory:"), - ("/prefix/", None, "/prefix/:memory:"), - ("/", "json", "/:memory:.json"), + ("/", None, "/%3Amemory%3A"), + ("/prefix/", None, "/prefix/%3Amemory%3A"), + ("/", "json", "/%3Amemory%3A.json"), ], ) def test_database(ds, base_url, format, expected): @@ -118,10 +118,10 @@ def test_database(ds, base_url, format, expected): @pytest.mark.parametrize( "base_url,name,format,expected", [ - ("/", "name", None, "/:memory:/name"), - ("/prefix/", "name", None, "/prefix/:memory:/name"), - ("/", "name", "json", "/:memory:/name.json"), - ("/", "name.json", "json", "/:memory:/name.json?_format=json"), + ("/", "name", None, "/%3Amemory%3A/name"), + ("/prefix/", "name", None, "/prefix/%3Amemory%3A/name"), + ("/", "name", "json", "/%3Amemory%3A/name.json"), + ("/", "name.json", "json", "/%3Amemory%3A/name.json?_format=json"), ], ) def test_table_and_query(ds, base_url, name, format, expected): @@ -137,9 +137,9 @@ def test_table_and_query(ds, base_url, name, format, expected): @pytest.mark.parametrize( "base_url,format,expected", [ - ("/", None, "/:memory:/facetable/1"), - ("/prefix/", None, "/prefix/:memory:/facetable/1"), - ("/", "json", "/:memory:/facetable/1.json"), + ("/", None, "/%3Amemory%3A/facetable/1"), + ("/prefix/", None, "/prefix/%3Amemory%3A/facetable/1"), + ("/", "json", "/%3Amemory%3A/facetable/1.json"), ], ) def test_row(ds, base_url, format, expected): From 0b9ac1b2e9c855f1b823a06a898891da87c720ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 Jan 2021 09:33:29 -0800 Subject: [PATCH 0264/1866] Release 0.54 Refs #509, #1091, #1150, #1151, #1166, #1167, #1178, #1181, #1182, #1184, #1185, #1186, #1187, #1194, #1198 --- datasette/version.py | 2 +- docs/changelog.rst | 54 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index b19423a9..8fb7217d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.54a0" +__version__ = "0.54" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index abc2f4f9..8fca312d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,16 +6,61 @@ Changelog .. _v0_54: -0.54 (2021-01-24) +0.54 (2021-01-25) ----------------- +The two big new features in this release are the ``_internal`` SQLite in-memory database storing details of all connected databases and tables, and support for JavaScript modules in plugins and additional scripts. +For additional commentary on this release, see `Datasette 0.54, the annotated release notes `__. +The _internal database +~~~~~~~~~~~~~~~~~~~~~~ +As part of ongoing work to help Datasette handle much larger numbers of connected databases and tables (see `Datasette Library `__) Datasette now maintains an in-memory SQLite database with details of all of the attached databases, tables, columns, indexes and foreign keys. (`#1150 `__) + +This will support future improvements such as a searchable, paginated homepage of all available tables. + +You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. + +Plugins can use these tables to introspect attached data in an efficient way. Plugin authors should note that this is not yet considered a stable interface, so any plugins that use this may need to make changes prior to Datasette 1.0 if the ``_internal`` table schemas change. + +Named in-memory database support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As part of the work building the ``_internal`` database, Datasette now supports named in-memory databases that can be shared across multiple connections. This allows plugins to create in-memory databases which will persist data for the lifetime of the Datasette server process. (`#1151 `__) + +The new ``memory_name=`` parameter to the :ref:`internals_database` can be used to create named, shared in-memory databases. + +JavaScript modules +~~~~~~~~~~~~~~~~~~ + +`JavaScript modules `__ were introduced in ECMAScript 2015 and provide native browser support for the ``import`` and ``export`` keywords. + +To use modules, JavaScript needs to be included in `` + diff --git a/datasette/templates/_codemirror_foot.html b/datasette/templates/_codemirror_foot.html index 4019d448..ee09cff1 100644 --- a/datasette/templates/_codemirror_foot.html +++ b/datasette/templates/_codemirror_foot.html @@ -23,6 +23,7 @@ window.onload = () => { editor.setValue(sqlFormatter.format(editor.getValue())); }) } + cmResize(editor, {resizableWidth: false}); } if (sqlFormat && readOnly) { const formatted = sqlFormatter.format(readOnly.innerHTML); From 42caabf7e9e6e4d69ef6dd7de16f2cd96bc79d5b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Feb 2021 09:35:41 -0800 Subject: [PATCH 0283/1866] Fixed typo --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 8ea5e79b..1291a875 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -79,7 +79,7 @@ Using pytest fixtures A common pattern for Datasette plugins is to create a fixture which sets up a temporary test database and wraps it in a Datasette instance. -Here's an example that uses the `sqlite-utils library `__ to populate a temporary test database. It also sets the title of that table using a simulated ``metadata.json`` congiguration: +Here's an example that uses the `sqlite-utils library `__ to populate a temporary test database. It also sets the title of that table using a simulated ``metadata.json`` configuration: .. code-block:: python From 726f781c50e88f557437f6490b8479c3d6fabfc2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Feb 2021 16:22:47 -0800 Subject: [PATCH 0284/1866] Fix for arraycontains bug, closes #1239 --- datasette/filters.py | 4 ++-- tests/test_filters.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 152a26b4..2b859d99 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -150,7 +150,7 @@ class Filters: "arraycontains", "array contains", """rowid in ( - select {t}.rowid from {t}, json_each({t}.{c}) j + select {t}.rowid from {t}, json_each([{t}].[{c}]) j where j.value = :{p} )""", '{c} contains "{v}"', @@ -159,7 +159,7 @@ class Filters: "arraynotcontains", "array does not contain", """rowid not in ( - select {t}.rowid from {t}, json_each({t}.{c}) j + select {t}.rowid from {t}, json_each([{t}].[{c}]) j where j.value = :{p} )""", '{c} does not contain "{v}"', diff --git a/tests/test_filters.py b/tests/test_filters.py index 75a779b9..f22b7b5c 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -56,6 +56,14 @@ import pytest # Not in, and JSON array not in ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]), ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]), + # JSON arraycontains + ( + (("Availability+Info__arraycontains", "yes"),), + [ + "rowid in (\n select table.rowid from table, json_each([table].[Availability+Info]) j\n where j.value = :p0\n )" + ], + ["yes"], + ), ], ) def test_build_where(args, expected_where, expected_params): From afed51b1e36cf275c39e71c7cb262d6c5bdbaa31 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Feb 2021 09:27:09 -0800 Subject: [PATCH 0285/1866] Note about where to find plugin examples, closes #1244 --- docs/writing_plugins.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index b43ecb27..6afee1c3 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -5,6 +5,8 @@ Writing plugins You can write one-off plugins that apply to just one Datasette instance, or you can write plugins which can be installed using ``pip`` and can be shipped to the Python Package Index (`PyPI `__) for other people to install. +Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 50 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. + .. _writing_plugins_one_off: Writing one-off plugins From cc6774cbaaba2359e0a92cfcc41ad988680075d6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 14:34:44 -0800 Subject: [PATCH 0286/1866] Upgrade httpx and remove xfail from tests, refs #1005 --- setup.py | 2 +- tests/test_api.py | 2 -- tests/test_html.py | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 34b6b396..15ee63fe 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ setup( "click-default-group~=1.2.2", "Jinja2>=2.10.3,<2.12.0", "hupper~=1.9", - "httpx>=0.15", + "httpx>=0.17", "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", diff --git a/tests/test_api.py b/tests/test_api.py index 0b5401d6..caf23329 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -810,7 +810,6 @@ def test_table_shape_object_compound_primary_key(app_client): assert {"a,b": {"pk1": "a", "pk2": "b", "content": "c"}} == response.json -@pytest.mark.xfail def test_table_with_slashes_in_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv?_shape=objects&_format=json" @@ -1286,7 +1285,6 @@ def test_row_format_in_querystring(app_client): assert [{"id": "1", "content": "hello"}] == response.json["rows"] -@pytest.mark.xfail def test_row_strange_table_name(app_client): response = app_client.get( "/fixtures/table%2Fwith%2Fslashes.csv/3.json?_shape=objects" diff --git a/tests/test_html.py b/tests/test_html.py index e21bd64d..3482ec35 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -158,7 +158,6 @@ def test_row_redirects_with_url_hash(app_client_with_hash): assert response.status == 200 -@pytest.mark.xfail def test_row_strange_table_name_with_url_hash(app_client_with_hash): response = app_client_with_hash.get( "/fixtures/table%2Fwith%2Fslashes.csv/3", allow_redirects=False @@ -552,7 +551,6 @@ def test_facets_persist_through_filter_form(app_client): ] -@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_classes", [ @@ -584,7 +582,6 @@ def test_css_classes_on_body(app_client, path, expected_classes): assert classes == expected_classes -@pytest.mark.xfail @pytest.mark.parametrize( "path,expected_considered", [ From 47eb885cc2c3aafa03645c330c6f597bee9b3b25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 19:44:04 -0800 Subject: [PATCH 0287/1866] JSON faceting now suggested even if column has blank strings, closes #1246 --- datasette/facets.py | 11 ++++++++--- tests/test_facets.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 207d819d..01628760 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -279,6 +279,7 @@ class ArrayFacet(Facet): suggested_facet_sql = """ select distinct json_type({column}) from ({sql}) + where {column} is not null and {column} != '' """.format( column=escape_sqlite(column), sql=self.sql ) @@ -298,9 +299,13 @@ class ArrayFacet(Facet): v[0] for v in await self.ds.execute( self.database, - "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format( - column=escape_sqlite(column), sql=self.sql - ), + ( + "select {column} from ({sql}) " + "where {column} is not null " + "and {column} != '' " + "and json_array_length({column}) > 0 " + "limit 100" + ).format(column=escape_sqlite(column), sql=self.sql), self.params, truncate=False, custom_time_limit=self.ds.setting( diff --git a/tests/test_facets.py b/tests/test_facets.py index 1e19dc3a..31518682 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1,3 +1,5 @@ +from datasette.app import Datasette +from datasette.database import Database from datasette.facets import ColumnFacet, ArrayFacet, DateFacet from datasette.utils.asgi import Request from datasette.utils import detect_json1 @@ -325,3 +327,23 @@ async def test_date_facet_results(app_client): "truncated": False, } } == buckets + + +@pytest.mark.asyncio +async def test_json_array_with_blanks_and_nulls(): + ds = Datasette([], memory=True) + db = ds.add_database(Database(ds, memory_name="test_json_array")) + await db.execute_write("create table foo(json_column text)", block=True) + for value in ('["a", "b", "c"]', '["a", "b"]', "", None): + await db.execute_write( + "insert into foo (json_column) values (?)", [value], block=True + ) + response = await ds.client.get("/test_json_array/foo.json") + data = response.json() + assert data["suggested_facets"] == [ + { + "name": "json_column", + "type": "array", + "toggle_url": "http://localhost/test_json_array/foo.json?_facet_array=json_column", + } + ] From 7c87532acc4e9d92caa1c4ee29a3446200928018 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Feb 2021 20:02:18 -0800 Subject: [PATCH 0288/1866] New .add_memory_database() method, closes #1247 --- datasette/app.py | 3 +++ docs/internals.rst | 29 ++++++++++++++++++++--------- tests/test_internals_database.py | 4 ++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index e3272c6e..02d432df 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -390,6 +390,9 @@ class Datasette: self.databases[name] = db return db + def add_memory_database(self, memory_name): + return self.add_database(Database(self, memory_name=memory_name)) + def remove_database(self, name): self.databases.pop(name) diff --git a/docs/internals.rst b/docs/internals.rst index 713f5d7d..e3bb83fd 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -273,7 +273,25 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` This will add a mutable database and serve it at ``/my-new-database``. -To create a shared in-memory database named ``statistics``, use the following: +``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: + +.. code-block:: python + + db = datasette.add_database(Database(datasette, memory_name="statistics")) + await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) + +.. _datasette_add_memory_database: + +.add_memory_database(name) +-------------------------- + +Adds a shared in-memory database with the specified name: + +.. code-block:: python + + datasette.add_memory_database("statistics") + +This is a shortcut for the following: .. code-block:: python @@ -284,14 +302,7 @@ To create a shared in-memory database named ``statistics``, use the following: memory_name="statistics" )) -This database will be served at ``/statistics``. - -``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: - -.. code-block:: python - - db = datasette.add_database(Database(datasette, memory_name="statistics")) - await db.execute_write("CREATE TABLE foo(id integer primary key)", block=True) +Using either of these pattern will result in the in-memory database being served at ``/statistics``. .. _datasette_remove_database: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 086f1a48..b60aaa8e 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -479,9 +479,9 @@ async def test_attached_databases(app_client_two_attached_databases_crossdb_enab async def test_database_memory_name(app_client): ds = app_client.ds foo1 = ds.add_database(Database(ds, memory_name="foo")) - foo2 = ds.add_database(Database(ds, memory_name="foo")) + foo2 = ds.add_memory_database("foo") bar1 = ds.add_database(Database(ds, memory_name="bar")) - bar2 = ds.add_database(Database(ds, memory_name="bar")) + bar2 = ds.add_memory_database("bar") for db in (foo1, foo2, bar1, bar2): table_names = await db.table_names() assert table_names == [] From 4f9a2f1f47dcf7e8561d68a8a07f5009a13cfdb3 Mon Sep 17 00:00:00 2001 From: David Boucha Date: Wed, 3 Mar 2021 22:46:10 -0700 Subject: [PATCH 0289/1866] Fix small typo (#1243) Thanks, @UtahDave --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 4e04ea1d..0f892f83 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -58,7 +58,7 @@ Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Data $ python3 -c 'import secrets; print(secrets.token_hex(32))' -This configuration will run Datasette against all database files contained in the ``/home/ubunt/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. +This configuration will run Datasette against all database files contained in the ``/home/ubuntu/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. You can start the Datasette process running using the following:: From d0fd833b8cdd97e1b91d0f97a69b494895d82bee Mon Sep 17 00:00:00 2001 From: Bob Whitelock Date: Sun, 7 Mar 2021 07:41:17 +0000 Subject: [PATCH 0290/1866] Add compile option to Dockerfile to fix failing test (fixes #696) (#1223) This test was failing when run inside the Docker container: `test_searchable[/fixtures/searchable.json?_search=te*+AND+do*&_searchmode=raw-expected_rows3]`, with this error: ``` def test_searchable(app_client, path, expected_rows): response = app_client.get(path) > assert expected_rows == response.json["rows"] E AssertionError: assert [[1, 'barry c...sel', 'puma']] == [] E Left contains 2 more items, first extra item: [1, 'barry cat', 'terry dog', 'panther'] E Full diff: E + [] E - [[1, 'barry cat', 'terry dog', 'panther'], E - [2, 'terry dog', 'sara weasel', 'puma']] ``` The issue was that the version of sqlite3 built inside the Docker container was built with FTS3 and FTS4 enabled, but without the `SQLITE_ENABLE_FTS3_PARENTHESIS` compile option passed, which adds support for using `AND` and `NOT` within `match` expressions (see https://sqlite.org/fts3.html#compiling_and_enabling_fts3_and_fts4 and https://www.sqlite.org/compile.html). Without this, the `AND` used in the search in this test was being interpreted as a literal string, and so no matches were found. Adding this compile option fixes this. Thanks, @bobwhitelock --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aba701ab..f4b14146 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt update \ RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \ - && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ + && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ && make && make install RUN wget "http://www.gaia-gis.it/gaia-sins/freexl-sources/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \ From a1bcd2fbe5e47bb431045f65eeceb5eb3a6718d5 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Pressac Date: Wed, 10 Mar 2021 19:26:39 +0100 Subject: [PATCH 0291/1866] Minor typo in IP adress (#1256) 127.0.01 replaced by 127.0.0.1 --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 0f892f83..48261b59 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -67,7 +67,7 @@ You can start the Datasette process running using the following:: You can confirm that Datasette is running on port 8000 like so:: - curl 127.0.01:8000/-/versions.json + curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. From 8e18c7943181f228ce5ebcea48deb59ce50bee1f Mon Sep 17 00:00:00 2001 From: Konstantin Baikov <4488943+kbaikov@users.noreply.github.com> Date: Thu, 11 Mar 2021 17:15:49 +0100 Subject: [PATCH 0292/1866] Use context manager instead of plain open (#1211) Context manager with open closes the files after usage. When the object is already a pathlib.Path i used read_text write_text functions In some cases pathlib.Path.open were used in context manager, it is basically the same as builtin open. Thanks, Konstantin Baikov! --- datasette/app.py | 13 ++++++------- datasette/cli.py | 13 +++++++------ datasette/publish/cloudrun.py | 6 ++++-- datasette/publish/heroku.py | 17 ++++++++++------- datasette/utils/__init__.py | 6 ++++-- setup.py | 3 ++- tests/conftest.py | 6 ++---- tests/fixtures.py | 5 +++-- tests/test_cli.py | 3 ++- tests/test_cli_serve_get.py | 3 ++- tests/test_docs.py | 8 ++++---- tests/test_package.py | 6 ++++-- tests/test_plugins.py | 3 ++- tests/test_publish_cloudrun.py | 32 ++++++++++++++++++++------------ tests/test_publish_heroku.py | 12 ++++++++---- tests/test_utils.py | 18 ++++++++++++------ update-docs-help.py | 2 +- 17 files changed, 93 insertions(+), 63 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 02d432df..f43ec205 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -212,7 +212,7 @@ class Datasette: and (config_dir / "inspect-data.json").exists() and not inspect_data ): - inspect_data = json.load((config_dir / "inspect-data.json").open()) + inspect_data = json.loads((config_dir / "inspect-data.json").read_text()) if immutables is None: immutable_filenames = [i["file"] for i in inspect_data.values()] immutables = [ @@ -269,7 +269,7 @@ class Datasette: if config_dir and (config_dir / "config.json").exists(): raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: - config = json.load((config_dir / "settings.json").open()) + config = json.loads((config_dir / "settings.json").read_text()) self._settings = dict(DEFAULT_SETTINGS, **(config or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note @@ -450,11 +450,10 @@ class Datasette: def app_css_hash(self): if not hasattr(self, "_app_css_hash"): - self._app_css_hash = hashlib.sha1( - open(os.path.join(str(app_root), "datasette/static/app.css")) - .read() - .encode("utf8") - ).hexdigest()[:6] + with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: + self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ + :6 + ] return self._app_css_hash async def get_canned_queries(self, database_name, actor): diff --git a/datasette/cli.py b/datasette/cli.py index 96a41740..2fa039a0 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -125,13 +125,13 @@ def cli(): @sqlite_extensions def inspect(files, inspect_file, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) - if inspect_file == "-": - out = sys.stdout - else: - out = open(inspect_file, "w") loop = asyncio.get_event_loop() inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions)) - out.write(json.dumps(inspect_data, indent=2)) + if inspect_file == "-": + sys.stdout.write(json.dumps(inspect_data, indent=2)) + else: + with open(inspect_file, "w") as fp: + fp.write(json.dumps(inspect_data, indent=2)) async def inspect_(files, sqlite_extensions): @@ -475,7 +475,8 @@ def serve( inspect_data = None if inspect_file: - inspect_data = json.load(open(inspect_file)) + with open(inspect_file) as fp: + inspect_data = json.load(fp) metadata_data = None if metadata: diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 7f9e89e2..bad223a1 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -141,9 +141,11 @@ def publish_subcommand(publish): if show_files: if os.path.exists("metadata.json"): print("=== metadata.json ===\n") - print(open("metadata.json").read()) + with open("metadata.json") as fp: + print(fp.read()) print("\n==== Dockerfile ====\n") - print(open("Dockerfile").read()) + with open("Dockerfile") as fp: + print(fp.read()) print("\n====================\n") image_id = f"gcr.io/{project}/{name}" diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index c0c70e12..19fe3fbe 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -171,9 +171,11 @@ def temporary_heroku_directory( os.chdir(tmp.name) if metadata_content: - open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) + with open("metadata.json", "w") as fp: + fp.write(json.dumps(metadata_content, indent=2)) - open("runtime.txt", "w").write("python-3.8.7") + with open("runtime.txt", "w") as fp: + fp.write("python-3.8.7") if branch: install = [ @@ -182,11 +184,11 @@ def temporary_heroku_directory( else: install = ["datasette"] + list(install) - open("requirements.txt", "w").write("\n".join(install)) + with open("requirements.txt", "w") as fp: + fp.write("\n".join(install)) os.mkdir("bin") - open("bin/post_compile", "w").write( - "datasette inspect --inspect-file inspect-data.json" - ) + with open("bin/post_compile", "w") as fp: + fp.write("datasette inspect --inspect-file inspect-data.json") extras = [] if template_dir: @@ -218,7 +220,8 @@ def temporary_heroku_directory( procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format( quoted_files=quoted_files, extras=" ".join(extras) ) - open("Procfile", "w").write(procfile_cmd) + with open("Procfile", "w") as fp: + fp.write(procfile_cmd) for path, filename in zip(file_paths, file_names): link_or_copy(path, os.path.join(tmp.name, filename)) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 47ca0551..1fedb69c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -428,8 +428,10 @@ def temporary_docker_directory( ) os.chdir(datasette_dir) if metadata_content: - open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) - open("Dockerfile", "w").write(dockerfile) + with open("metadata.json", "w") as fp: + fp.write(json.dumps(metadata_content, indent=2)) + with open("Dockerfile", "w") as fp: + fp.write(dockerfile) for path, filename in zip(file_paths, file_names): link_or_copy(path, os.path.join(datasette_dir, filename)) if template_dir: diff --git a/setup.py b/setup.py index 15ee63fe..3540e30a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ def get_version(): os.path.dirname(os.path.abspath(__file__)), "datasette", "version.py" ) g = {} - exec(open(path).read(), g) + with open(path) as fp: + exec(fp.read(), g) return g["__version__"] diff --git a/tests/conftest.py b/tests/conftest.py index b00ea006..ad3eb9f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,10 +75,8 @@ def check_permission_actions_are_documented(): from datasette.plugins import pm content = ( - (pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst") - .open() - .read() - ) + pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" + ).read_text() permissions_re = re.compile(r"\.\. _permissions_([^\s:]+):") documented_permission_actions = set(permissions_re.findall(content)).union( UNDOCUMENTED_PERMISSIONS diff --git a/tests/fixtures.py b/tests/fixtures.py index 30113ff2..2fd8e9cb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -789,7 +789,8 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): conn.executescript(GENERATED_COLUMNS_SQL) print(f"Test tables written to {db_filename}") if metadata: - open(metadata, "w").write(json.dumps(METADATA, indent=4)) + with open(metadata, "w") as fp: + fp.write(json.dumps(METADATA, indent=4)) print(f"- metadata written to {metadata}") if plugins_path: path = pathlib.Path(plugins_path) @@ -798,7 +799,7 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): test_plugins = pathlib.Path(__file__).parent / "plugins" for filepath in test_plugins.glob("*.py"): newpath = path / filepath.name - newpath.write_text(filepath.open().read()) + newpath.write_text(filepath.read_text()) print(f" Wrote plugin: {newpath}") if extra_db_filename: if pathlib.Path(extra_db_filename).exists(): diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ddd32f6..e094ccb6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -49,7 +49,8 @@ def test_inspect_cli_writes_to_file(app_client): cli, ["inspect", "fixtures.db", "--inspect-file", "foo.json"] ) assert 0 == result.exit_code, result.output - data = json.load(open("foo.json")) + with open("foo.json") as fp: + data = json.load(fp) assert ["fixtures"] == list(data.keys()) diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index aaa692e5..90fbfe3b 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -14,7 +14,8 @@ def test_serve_with_get(tmp_path_factory): @hookimpl def startup(datasette): - open("{}", "w").write("hello") + with open("{}", "w") as fp: + fp.write("hello") """.format( str(plugins_dir / "hello.txt") ), diff --git a/tests/test_docs.py b/tests/test_docs.py index 44b0810a..efd267b9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -19,13 +19,13 @@ def get_headings(content, underline="-"): def get_labels(filename): - content = (docs_path / filename).open().read() + content = (docs_path / filename).read_text() return set(label_re.findall(content)) @pytest.fixture(scope="session") def settings_headings(): - return get_headings((docs_path / "settings.rst").open().read(), "~") + return get_headings((docs_path / "settings.rst").read_text(), "~") @pytest.mark.parametrize("setting", app.SETTINGS) @@ -43,7 +43,7 @@ def test_settings_are_documented(settings_headings, setting): ), ) def test_help_includes(name, filename): - expected = open(str(docs_path / filename)).read() + expected = (docs_path / filename).read_text() runner = CliRunner() result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) actual = f"$ datasette {name} --help\n\n{result.output}" @@ -55,7 +55,7 @@ def test_help_includes(name, filename): @pytest.fixture(scope="session") def plugin_hooks_content(): - return (docs_path / "plugin_hooks.rst").open().read() + return (docs_path / "plugin_hooks.rst").read_text() @pytest.mark.parametrize( diff --git a/tests/test_package.py b/tests/test_package.py index 3248b3a4..bb939643 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -32,7 +32,8 @@ def test_package(mock_call, mock_which): capture = CaptureDockerfile() mock_call.side_effect = capture with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"]) assert 0 == result.exit_code mock_call.assert_has_calls([mock.call(["docker", "build", "."])]) @@ -47,7 +48,8 @@ def test_package_with_port(mock_call, mock_which): mock_call.side_effect = capture runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["package", "test.db", "-p", "8080", "--secret", "sekrit"] ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 715c7c17..ee6f1efa 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -225,7 +225,8 @@ def test_plugin_config_env_from_list(app_client): def test_plugin_config_file(app_client): - open(TEMP_PLUGIN_SECRET_FILE, "w").write("FROM_FILE") + with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: + fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == app_client.ds.plugin_config("file-plugin") # Ensure secrets aren't visible in /-/metadata.json metadata = app_client.get("/-/metadata.json") diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 2ef90705..7881ebae 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -11,7 +11,8 @@ def test_publish_cloudrun_requires_gcloud(mock_which): mock_which.return_value = False runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) assert result.exit_code == 1 assert "Publishing to Google Cloud requires gcloud" in result.output @@ -40,7 +41,8 @@ def test_publish_cloudrun_prompts_for_service( mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" ) @@ -81,7 +83,8 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] ) @@ -120,7 +123,8 @@ def test_publish_cloudrun_memory( mock_which.return_value = True runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "cloudrun", "test.db", "--service", "test", "--memory", memory], @@ -152,17 +156,19 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") - open("metadata.yml", "w").write( - textwrap.dedent( - """ + with open("test.db", "w") as fp: + fp.write("data") + with open("metadata.yml", "w") as fp: + fp.write( + textwrap.dedent( + """ title: Hello from metadata YAML plugins: datasette-auth-github: foo: bar """ - ).strip() - ) + ).strip() + ) result = runner.invoke( cli.cli, [ @@ -228,7 +234,8 @@ def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ @@ -295,7 +302,8 @@ def test_publish_cloudrun_extra_options( runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index c7a38031..c011ab43 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -8,7 +8,8 @@ def test_publish_heroku_requires_heroku(mock_which): mock_which.return_value = False runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) assert result.exit_code == 1 assert "Publishing to Heroku requires heroku" in result.output @@ -22,7 +23,8 @@ def test_publish_heroku_installs_plugin(mock_call, mock_check_output, mock_which mock_check_output.side_effect = lambda s: {"['heroku', 'plugins']": b""}[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("t.db", "w").write("data") + with open("t.db", "w") as fp: + fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") assert 0 != result.exit_code mock_check_output.assert_has_calls( @@ -54,7 +56,8 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): }[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, ["publish", "heroku", "test.db", "--tar", "gtar"] ) @@ -88,7 +91,8 @@ def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which) }[repr(s)] runner = CliRunner() with runner.isolated_filesystem(): - open("test.db", "w").write("data") + with open("test.db", "w") as fp: + fp.write("data") result = runner.invoke( cli.cli, [ diff --git a/tests/test_utils.py b/tests/test_utils.py index 56306339..ecef6f7a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -232,7 +232,8 @@ def test_to_css_class(s, expected): def test_temporary_docker_directory_uses_hard_link(): with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") # Default usage of this should use symlink with utils.temporary_docker_directory( files=["hello"], @@ -249,7 +250,8 @@ def test_temporary_docker_directory_uses_hard_link(): secret="secret", ) as temp_docker: hello = os.path.join(temp_docker, "hello") - assert "world" == open(hello).read() + with open(hello) as fp: + assert "world" == fp.read() # It should be a hard link assert 2 == os.stat(hello).st_nlink @@ -260,7 +262,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): mock_link.side_effect = OSError with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") # Default usage of this should use symlink with utils.temporary_docker_directory( files=["hello"], @@ -277,7 +280,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): secret=None, ) as temp_docker: hello = os.path.join(temp_docker, "hello") - assert "world" == open(hello).read() + with open(hello) as fp: + assert "world" == fp.read() # It should be a copy, not a hard link assert 1 == os.stat(hello).st_nlink @@ -285,7 +289,8 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link): def test_temporary_docker_directory_quotes_args(): with tempfile.TemporaryDirectory() as td: os.chdir(td) - open("hello", "w").write("world") + with open("hello", "w") as fp: + fp.write("world") with utils.temporary_docker_directory( files=["hello"], name="t", @@ -301,7 +306,8 @@ def test_temporary_docker_directory_quotes_args(): secret="secret", ) as temp_docker: df = os.path.join(temp_docker, "Dockerfile") - df_contents = open(df).read() + with open(df) as fp: + df_contents = fp.read() assert "'$PWD'" in df_contents assert "'--$HOME'" in df_contents assert "ENV DATASETTE_SECRET 'secret'" in df_contents diff --git a/update-docs-help.py b/update-docs-help.py index 3a192575..292d1dcd 100644 --- a/update-docs-help.py +++ b/update-docs-help.py @@ -18,7 +18,7 @@ def update_help_includes(): result = runner.invoke(cli, name.split() + ["--help"], terminal_width=88) actual = f"$ datasette {name} --help\n\n{result.output}" actual = actual.replace("Usage: cli ", "Usage: datasette ") - open(docs_path / filename, "w").write(actual) + (docs_path / filename).write_text(actual) if __name__ == "__main__": From c4f1ec7f33fd7d5b93f0f895dafb5351cc3bfc5b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 20 Mar 2021 14:32:23 -0700 Subject: [PATCH 0293/1866] Documentation for Response.asgi_send(), closes #1266 --- docs/internals.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index e3bb83fd..18032406 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -138,6 +138,28 @@ Each of these responses will use the correct corresponding content-type - ``text Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above. +.. _internals_response_asgi_send: + +Returning a response with .asgi_send(send) +------------------------------------------ + + +In most cases you will return ``Response`` objects from your own view functions. You can also use a ``Response`` instance to respond at a lower level via ASGI, for example if you are writing code that uses the :ref:`plugin_asgi_wrapper` hook. + +Create a ``Response`` object and then use ``await response.asgi_send(send)``, passing the ASGI ``send`` function. For example: + +.. code-block:: python + + async def require_authorization(scope, recieve, send): + response = Response.text( + "401 Authorization Required", + headers={ + "www-authenticate": 'Basic realm="Datasette", charset="UTF-8"' + }, + status=401, + ) + await response.asgi_send(send) + .. _internals_response_set_cookie: Setting cookies with response.set_cookie() From 6ad544df5e6bd027a8e27317041e6168aee07459 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Mar 2021 09:19:41 -0700 Subject: [PATCH 0294/1866] Fixed master -> main in a bunch of places, mainly docs --- datasette/cli.py | 2 +- datasette/publish/common.py | 2 +- datasette/templates/patterns.html | 16 ++++++++-------- docs/contributing.rst | 2 +- docs/custom_templates.rst | 2 +- docs/datasette-package-help.txt | 2 +- docs/datasette-publish-cloudrun-help.txt | 2 +- docs/datasette-publish-heroku-help.txt | 2 +- docs/plugin_hooks.rst | 4 ++-- docs/publish.rst | 4 ++-- docs/spatialite.rst | 2 +- tests/fixtures.py | 4 ++-- tests/test_html.py | 9 ++++----- 13 files changed, 26 insertions(+), 27 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 2fa039a0..42b5c115 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -191,7 +191,7 @@ def plugins(all, plugins_dir): help="Path to JSON/YAML file containing metadata to publish", ) @click.option("--extra-options", help="Extra options to pass to datasette serve") -@click.option("--branch", help="Install datasette from a GitHub branch e.g. master") +@click.option("--branch", help="Install datasette from a GitHub branch e.g. main") @click.option( "--template-dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), diff --git a/datasette/publish/common.py b/datasette/publish/common.py index b6570290..29665eb3 100644 --- a/datasette/publish/common.py +++ b/datasette/publish/common.py @@ -19,7 +19,7 @@ def add_common_publish_arguments_and_options(subcommand): "--extra-options", help="Extra options to pass to datasette serve" ), click.option( - "--branch", help="Install datasette from a GitHub branch e.g. master" + "--branch", help="Install datasette from a GitHub branch e.g. main" ), click.option( "--template-dir", diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 984c1bf6..3f9b5a16 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -70,10 +70,10 @@

    Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -118,10 +118,10 @@

    Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -177,10 +177,10 @@

    Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: @@ -478,10 +478,10 @@

    Powered by Datasette · Data license: - Apache License 2.0 + Apache License 2.0 · Data source: - + tests/fixtures.py · About: diff --git a/docs/contributing.rst b/docs/contributing.rst index 2cf641fd..7e16280b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -10,7 +10,7 @@ This document describes how to contribute to Datasette core. You can also contri General guidelines ------------------ -* **master should always be releasable**. Incomplete features should live in branches. This ensures that any small bug fixes can be quickly released. +* **main should always be releasable**. Incomplete features should live in branches. This ensures that any small bug fixes can be quickly released. * **The ideal commit** should bundle together the implementation, unit tests and associated documentation updates. The commit message should link to an associated issue. * **New plugin hooks** should only be shipped if accompanied by a separate release of a non-demo plugin that uses them. diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index a7236873..efb5b842 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -303,7 +303,7 @@ from the default template. The ``_table.html`` template is included by both the row and the table pages, and a list of rows. The default ``_table.html`` template renders them as an -HTML template and `can be seen here `_. +HTML template and `can be seen here `_. You can provide a custom template that applies to all of your databases and tables, or you can provide custom templates for specific tables using the diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt index ab5cf8a3..5f5ce070 100644 --- a/docs/datasette-package-help.txt +++ b/docs/datasette-package-help.txt @@ -10,7 +10,7 @@ Options: -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master + --branch TEXT Install datasette from a GitHub branch e.g. main --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index 8cf293d9..c706d921 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -5,7 +5,7 @@ Usage: datasette publish cloudrun [OPTIONS] [FILES]... Options: -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master + --branch TEXT Install datasette from a GitHub branch e.g. main --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index 991bd8f4..c4b852de 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -5,7 +5,7 @@ Usage: datasette publish heroku [OPTIONS] [FILES]... Options: -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve - --branch TEXT Install datasette from a GitHub branch e.g. master + --branch TEXT Install datasette from a GitHub branch e.g. main --template-dir DIRECTORY Path to directory containing custom templates --plugins-dir DIRECTORY Path to directory containing custom plugins --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 23e57278..0a176add 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -304,7 +304,7 @@ publish_subcommand(publish) This hook allows you to create new providers for the ``datasette publish`` command. Datasette uses this hook internally to implement the default ``now`` and ``heroku`` subcommands, so you can read -`their source `_ +`their source `_ to see examples of this hook in action. Let's say you want to build a plugin that adds a ``datasette publish my_hosting_provider --api_key=xxx mydatabase.db`` publish command. Your implementation would start like this: @@ -641,7 +641,7 @@ Each Facet subclass implements a new type of facet operation. The class should l return facet_results, facets_timed_out -See `datasette/facets.py `__ for examples of how these classes can work. +See `datasette/facets.py `__ for examples of how these classes can work. The plugin hook can then be used to register the new facet class like this: diff --git a/docs/publish.rst b/docs/publish.rst index d5015e21..780933fc 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -78,7 +78,7 @@ Publishing to Vercel pip install datasette-publish-vercel datasette publish vercel mydatabase.db --project my-database-project -Not every feature is supported: consult the `datasette-publish-vercel README `__ for more details. +Not every feature is supported: consult the `datasette-publish-vercel README `__ for more details. .. _publish_fly: @@ -92,7 +92,7 @@ Publishing to Fly pip install datasette-publish-fly datasette publish fly mydatabase.db -Consult the `datasette-publish-fly README `__ for more details. +Consult the `datasette-publish-fly README `__ for more details. .. _publish_custom_metadata_and_plugins: diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 0871d72d..234d97e5 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -53,7 +53,7 @@ If you are unsure of the location of the module, try running ``locate mod_spatia Building SpatiaLite from source ------------------------------- -The packaged versions of SpatiaLite usually provide SpatiaLite 4.3.0a. For an example of how to build the most recent unstable version, 4.4.0-RC0 (which includes the powerful `VirtualKNN module `_), take a look at the `Datasette Dockerfile `_. +The packaged versions of SpatiaLite usually provide SpatiaLite 4.3.0a. For an example of how to build the most recent unstable version, 4.4.0-RC0 (which includes the powerful `VirtualKNN module `_), take a look at the `Datasette Dockerfile `_. Spatial indexing latitude/longitude columns =========================================== diff --git a/tests/fixtures.py b/tests/fixtures.py index 2fd8e9cb..0a721d3a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -283,9 +283,9 @@ METADATA = { "title": "Datasette Fixtures", "description_html": 'An example SQLite database demonstrating Datasette. Sign in as root user', "license": "Apache License 2.0", - "license_url": "https://github.com/simonw/datasette/blob/master/LICENSE", + "license_url": "https://github.com/simonw/datasette/blob/main/LICENSE", "source": "tests/fixtures.py", - "source_url": "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", + "source_url": "https://github.com/simonw/datasette/blob/main/tests/fixtures.py", "about": "About Datasette", "about_url": "https://github.com/simonw/datasette", "extra_css_urls": ["/static/extra-css-urls.css"], diff --git a/tests/test_html.py b/tests/test_html.py index 3482ec35..9e86ebc2 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1093,12 +1093,11 @@ def assert_footer_links(soup): assert "About Datasette" == about_link.text.strip() assert "https://datasette.io/" == datasette_link["href"] assert ( - "https://github.com/simonw/datasette/blob/master/tests/fixtures.py" + "https://github.com/simonw/datasette/blob/main/tests/fixtures.py" == source_link["href"] ) assert ( - "https://github.com/simonw/datasette/blob/master/LICENSE" - == license_link["href"] + "https://github.com/simonw/datasette/blob/main/LICENSE" == license_link["href"] ) assert "https://github.com/simonw/datasette" == about_link["href"] @@ -1513,8 +1512,8 @@ def test_base_url_config(app_client_base_url_prefix, path): and href not in { "https://datasette.io/", - "https://github.com/simonw/datasette/blob/master/LICENSE", - "https://github.com/simonw/datasette/blob/master/tests/fixtures.py", + "https://github.com/simonw/datasette/blob/main/LICENSE", + "https://github.com/simonw/datasette/blob/main/tests/fixtures.py", "/login-as-root", # Only used for the latest.datasette.io demo } and not href.startswith("https://plugin-example.datasette.io/") From 5fd02890650db790b2ffdb90eb9f78f8e0639c37 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Mar 2021 21:27:40 -0700 Subject: [PATCH 0295/1866] Build Dockerfile with SpatiaLite 5, refs #1249 --- .dockerignore | 2 ++ .github/workflows/publish.yml | 13 +++++---- Dockerfile | 55 ++++++++++++----------------------- 3 files changed, 29 insertions(+), 41 deletions(-) diff --git a/.dockerignore b/.dockerignore index 490f509e..5078bf47 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,5 @@ build dist scratchpad venv +*.db +*.sqlite diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c1909bbe..a3b29dd7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -66,8 +66,11 @@ jobs: DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PASS: ${{ secrets.DOCKER_PASS }} run: |- - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile -t $REPO:${GITHUB_REF#refs/tags/} . - docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push $REPO + sleep 60 # Give PyPI time to make the new release available + docker login -u $DOCKER_USER -p $DOCKER_PASS + export REPO=datasetteproject/datasette + docker build -f Dockerfile \ + -t $REPO:${GITHUB_REF#refs/tags/} \ + --build-arg VERSION=${GITHUB_REF#refs/tags/} . + docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest + docker push $REPO diff --git a/Dockerfile b/Dockerfile index f4b14146..8193700d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,25 @@ -FROM python:3.7.10-slim-stretch as build +FROM python:3.9.2-slim-buster as build -# Setup build dependencies -RUN apt update \ -&& apt install -y python3-dev build-essential wget libxml2-dev libproj-dev libgeos-dev libsqlite3-dev zlib1g-dev pkg-config git \ - && apt clean +# Version of Datasette to install, e.g. 0.55 +# docker build . -t datasette --build-arg VERSION=0.55 +ARG VERSION +# software-properties-common provides add-apt-repository +# which we need in order to install a more recent release +# of libsqlite3-mod-spatialite from the sid distribution +RUN apt-get update && \ + apt-get -y --no-install-recommends install software-properties-common && \ + add-apt-repository "deb http://httpredir.debian.org/debian sid main" && \ + apt-get update && \ + apt-get -t sid install -y --no-install-recommends libsqlite3-mod-spatialite && \ + apt-get remove -y software-properties-common && \ + apt clean && \ + rm -rf /var/lib/apt && \ + rm -rf /var/lib/dpkg -RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \ - && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \ - && make && make install - -RUN wget "http://www.gaia-gis.it/gaia-sins/freexl-sources/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \ - && cd freexl-1.0.5 && ./configure && make && make install - -RUN wget "http://www.gaia-gis.it/gaia-sins/libspatialite-sources/libspatialite-4.4.0-RC0.tar.gz" && tar zxf libspatialite-4.4.0-RC0.tar.gz \ - && cd libspatialite-4.4.0-RC0 && ./configure && make && make install - -RUN wget "http://www.gaia-gis.it/gaia-sins/readosm-sources/readosm-1.1.0.tar.gz" && tar zxf readosm-1.1.0.tar.gz && cd readosm-1.1.0 && ./configure && make && make install - -RUN wget "http://www.gaia-gis.it/gaia-sins/spatialite-tools-sources/spatialite-tools-4.4.0-RC0.tar.gz" && tar zxf spatialite-tools-4.4.0-RC0.tar.gz \ - && cd spatialite-tools-4.4.0-RC0 && ./configure && make && make install - - -# Add local code to the image instead of fetching from pypi. -COPY . /datasette - -RUN pip install /datasette - -FROM python:3.7.10-slim-stretch - -# Copy python dependencies and spatialite libraries -COPY --from=build /usr/local/lib/ /usr/local/lib/ -# Copy executables -COPY --from=build /usr/local/bin /usr/local/bin -# Copy spatial extensions -COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu - -ENV LD_LIBRARY_PATH=/usr/local/lib +RUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \ + find /usr/local/lib -name '__pycache__' | xargs rm -r && \ + rm -rf /root/.cache/pip EXPOSE 8001 CMD ["datasette"] From 8ebdcc916d556f7fb7fc2bbbb56904a6d8e1936c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Mar 2021 21:33:15 -0700 Subject: [PATCH 0296/1866] Remove obsolete note about building SpatiaLite from source, refs #1249 --- docs/spatialite.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 234d97e5..985954de 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -50,11 +50,6 @@ Depending on your distribution, you should be able to run Datasette something li If you are unsure of the location of the module, try running ``locate mod_spatialite`` and see what comes back. -Building SpatiaLite from source -------------------------------- - -The packaged versions of SpatiaLite usually provide SpatiaLite 4.3.0a. For an example of how to build the most recent unstable version, 4.4.0-RC0 (which includes the powerful `VirtualKNN module `_), take a look at the `Datasette Dockerfile `_. - Spatial indexing latitude/longitude columns =========================================== From 3fcfc8513465339ac5f055296cbb67f5262af02b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 27 Mar 2021 09:16:42 -0700 Subject: [PATCH 0297/1866] Fix links in SpatiaLite tutorial, closes #1278 --- docs/spatialite.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 985954de..d1b300b2 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -88,7 +88,7 @@ In the above example, the resulting index will be called ``idx_museums_point_geo select * from idx_museums_point_geom limit 10; -Here's a live example: `timezones-api.now.sh/timezones/idx_timezones_Geometry `_ +Here's a live example: `timezones-api.datasette.io/timezones/idx_timezones_Geometry `_ +--------+----------------------+----------------------+---------------------+---------------------+ | pkid | xmin | xmax | ymin | ymax | From 48d5e0e6ac8975cfd869d4e8c69c64ca0c65e29e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 16:44:29 -0700 Subject: [PATCH 0298/1866] Fix for no such table: pragma_database_list, refs #1276 --- datasette/database.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 3579cce9..9f3bbddc 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -247,10 +247,12 @@ class Database: return Path(self.path).stat().st_mtime_ns async def attached_databases(self): - results = await self.execute( - "select seq, name, file from pragma_database_list() where seq > 0" - ) - return [AttachedDatabase(*row) for row in results.rows] + # This used to be: + # select seq, name, file from pragma_database_list() where seq > 0 + # But SQLite prior to 3.16.0 doesn't support pragma functions + results = await self.execute("PRAGMA database_list;") + # {'seq': 0, 'name': 'main', 'file': ''} + return [AttachedDatabase(*row) for row in results.rows if row["seq"] > 0] async def table_exists(self, table): results = await self.execute( From c96a3826cf50cb347f6a415b56d8105ba6d8dcb0 Mon Sep 17 00:00:00 2001 From: vincent d warmerdam Date: Mon, 29 Mar 2021 02:11:55 +0200 Subject: [PATCH 0299/1866] Added `--app` to fly install command. (#1279) --- docs/publish.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/publish.rst b/docs/publish.rst index 780933fc..cbd18a00 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -90,7 +90,7 @@ Publishing to Fly :: pip install datasette-publish-fly - datasette publish fly mydatabase.db + datasette publish fly mydatabase.db --app="my-app" Consult the `datasette-publish-fly README `__ for more details. From e72397d65b06b019521b6411243687464ac8d8ca Mon Sep 17 00:00:00 2001 From: Bob Whitelock Date: Mon, 29 Mar 2021 01:14:04 +0100 Subject: [PATCH 0300/1866] Add styling to lists within table cells (fixes #1141) (#1252) This overrides the Datasette reset (see https://github.com/simonw/datasette/blob/d0fd833b8cdd97e1b91d0f97a69b494895d82bee/datasette/static/app.css#L35-L38), to add back the default styling of list items displayed within Datasette table cells. --- datasette/static/app.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 9e498ab9..fad11a3a 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -452,6 +452,10 @@ table a:link { margin-left: -10%; font-size: 0.8em; } +.rows-and-columns td ol,ul { + list-style: initial; + list-style-position: inside; +} a.blob-download { display: inline-block; } From f92d823766872a6fd7e76c5249a6b2de1ab0f447 Mon Sep 17 00:00:00 2001 From: Campbell Allen Date: Mon, 29 Mar 2021 01:17:31 +0100 Subject: [PATCH 0301/1866] ensure immutable databses when starting in configuration directory mode with (#1229) * check if immutables is empty list of None * update docs on how to create the inspect-data.json --- datasette/app.py | 2 +- docs/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f43ec205..6a7a6c6d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -213,7 +213,7 @@ class Datasette: and not inspect_data ): inspect_data = json.loads((config_dir / "inspect-data.json").read_text()) - if immutables is None: + if not immutables: immutable_filenames = [i["file"] for i in inspect_data.values()] immutables = [ f for f in self.files if Path(f).name in immutable_filenames diff --git a/docs/settings.rst b/docs/settings.rst index f2467aa4..b4c8a50e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -48,7 +48,7 @@ The files that can be included in this directory are as follows. All are optiona * ``*.db`` - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well -* ``inspect-data.json`` - the result of running ``datasette inspect`` - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running +* ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs * ``templates/`` - a directory containing :ref:`customization_custom_templates` * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off` From d579fcf4f713f98c7365453ce94f36b91ce98c98 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 17:20:55 -0700 Subject: [PATCH 0302/1866] Applied some fixes suggested by @withshubh in #1260 --- datasette/app.py | 4 ++-- tests/plugins/my_plugin.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 6a7a6c6d..ee816426 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -317,7 +317,7 @@ class Datasette: loader=template_loader, autoescape=True, enable_async=True ) self.jinja_env.filters["escape_css_string"] = escape_css_string - self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u) + self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member @@ -767,7 +767,7 @@ class Datasette: hook_renderers = [] # pylint: disable=no-member for hook in pm.hook.register_output_renderer(datasette=self): - if type(hook) == list: + if type(hook) is list: hook_renderers += hook else: hook_renderers.append(hook) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 8d192d28..26d06091 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -196,7 +196,7 @@ def permission_allowed(actor, action): elif action == "this_is_denied": return False elif action == "view-database-download": - return (actor and actor.get("can_download")) or None + return actor.get("can_download") if actor else None @hookimpl From af5a7f1c09f6a902bb2a25e8edf39c7034d2e5de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 17:41:12 -0700 Subject: [PATCH 0303/1866] Release 0.56 Refs #1005, #1031, #1141, #1229, #1236, #1239, #1246, #1247, #1252, #1266, #1276, #1278 --- datasette/version.py | 2 +- docs/changelog.rst | 18 ++++++++++++++++++ docs/internals.rst | 1 - 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 78eaa333..4dcf73b0 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.55" +__version__ = "0.56" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index eda87dbf..756badce 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,24 @@ Changelog ========= +.. _v0_56: + +0.56 (2021-03-28) +----------------- + +Documentation improvements, bug fixes and support for SpatiaLite 5. + +- The SQL editor can now be resized by dragging a handle. (:issue:`1236`) +- Fixed a bug with JSON faceting and the ``__arraycontains`` filter caused by tables with spaces in their names. (:issue:`1239`) +- Upgraded ``httpx`` dependency. (:issue:`1005`) +- JSON faceting is now suggested even if a column contains blank strings. (:issue:`1246`) +- New :ref:`datasette.add_memory_database() ` method. (:issue:`1247`) +- The :ref:`Response.asgi_send() ` method is now documented. (:issue:`1266`) +- The official Datasette Docker image now bundles SpatiaLite version 5. (:issue:`1278`) +- Fixed a ``no such table: pragma_database_list`` bug when running Datasette against SQLite versions prior to SQLite 3.16.0. (:issue:`1276`) +- HTML lists displayed in table cells are now styled correctly. Thanks, Bob Whitelock. (:issue:`1141`, `#1252 `__) +- Configuration directory mode now correctly serves immutable databases that are listed in ``inspect-data.json``. Thanks Campbell Allen and Frankie Robertson. (`#1031 `__, `#1229 `__) + .. _v0_55: 0.55 (2021-02-18) diff --git a/docs/internals.rst b/docs/internals.rst index 18032406..72c86083 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -143,7 +143,6 @@ Each of the helper methods take optional ``status=`` and ``headers=`` arguments, Returning a response with .asgi_send(send) ------------------------------------------ - In most cases you will return ``Response`` objects from your own view functions. You can also use a ``Response`` instance to respond at a lower level via ASGI, for example if you are writing code that uses the :ref:`plugin_asgi_wrapper` hook. Create a ``Response`` object and then use ``await response.asgi_send(send)``, passing the ASGI ``send`` function. For example: From 13fd9bdf01451decd55e1cbbd4017c0e5d0522e7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 18:07:49 -0700 Subject: [PATCH 0304/1866] docker push --all-tags, refs #1281 --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a3b29dd7..ad1e794d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,4 +73,4 @@ jobs: -t $REPO:${GITHUB_REF#refs/tags/} \ --build-arg VERSION=${GITHUB_REF#refs/tags/} . docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push $REPO + docker push --all-tags $REPO From 849c4f06ea766ccdb664eab4e82b80be574a0f03 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 18:35:56 -0700 Subject: [PATCH 0305/1866] Workflow for manually pushing a Docker tag, refs #1281 --- .github/workflows/push_docker_tag.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/push_docker_tag.yml diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml new file mode 100644 index 00000000..02391972 --- /dev/null +++ b/.github/workflows/push_docker_tag.yml @@ -0,0 +1,26 @@ +name: Push specific Docker tag + +on: + workflow_dispatch: + inputs: + version_tag: + description: Tag to build and push + +jobs: + deploy_docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build and push to Docker Hub + env: + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} + VERSION_TAG: ${{ github.event.inputs.version_tag }} + run: |- + docker login -u $DOCKER_USER -p $DOCKER_PASS + export REPO=datasetteproject/datasette + docker build -f Dockerfile \ + -t $REPO:${VERSION_TAG} \ + --build-arg VERSION=${VERSION_TAG} . + docker tag $REPO:${VERSION_TAG} + docker push $REPO From 8291065b13bf2a4af27d61a971a9ba96aff59417 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 18:39:02 -0700 Subject: [PATCH 0306/1866] Hopeful fix for Docker tag error, refs #1281 --- .github/workflows/push_docker_tag.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index 02391972..9a3969f0 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -22,5 +22,4 @@ jobs: docker build -f Dockerfile \ -t $REPO:${VERSION_TAG} \ --build-arg VERSION=${VERSION_TAG} . - docker tag $REPO:${VERSION_TAG} - docker push $REPO + docker push $REPO:${VERSION_TAG} From 0486303b60ce2784fd2e2ecdbecf304b7d6e6659 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 28 Mar 2021 18:42:42 -0700 Subject: [PATCH 0307/1866] Explicitly push version tag, refs #1281 --- .github/workflows/publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad1e794d..90fa4505 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,4 +73,5 @@ jobs: -t $REPO:${GITHUB_REF#refs/tags/} \ --build-arg VERSION=${GITHUB_REF#refs/tags/} . docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push --all-tags $REPO + docker push $REPO:${VERSION_TAG} + docker push $REPO:latest From 7b1a9a1999eb9326ce8ec830d75ac200e5279c46 Mon Sep 17 00:00:00 2001 From: Marjorie Roswell Date: Mon, 29 Mar 2021 15:57:34 -0400 Subject: [PATCH 0308/1866] Fix little typo (#1282) --- datasette/static/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index fad11a3a..4c41ea98 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -97,7 +97,7 @@ p { margin-bottom: 0.75rem; } .context-text { - /* for accessibility and hiden from sight */ + /* for accessibility and hidden from sight */ text-indent: -999em; display: block; width:0; From 87b583a128986982552421d2510e467e74ac5046 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 2 Apr 2021 13:20:51 -0700 Subject: [PATCH 0309/1866] Clearer help text for --reload Immutable databases are not commonly used, but it's useful to clarify that --reload will pick up on changes to metadata. --- README.md | 2 +- datasette/cli.py | 2 +- docs/datasette-serve-help.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a4fe36c0..4f3c9a94 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Now visiting http://localhost:8001/History/downloads will show you a web interfa allowed. Use 0.0.0.0 to listen to all IPs and allow access from other machines. -p, --port INTEGER Port for server, defaults to 8001 - --reload Automatically reload if database or code change + --reload Automatically reload if code or metadata change detected - useful for development --cors Enable CORS by serving Access-Control-Allow- Origin: * diff --git a/datasette/cli.py b/datasette/cli.py index 42b5c115..71bbc353 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -336,7 +336,7 @@ def uninstall(packages, yes): @click.option( "--reload", is_flag=True, - help="Automatically reload if database or code change detected - useful for development", + help="Automatically reload if code or metadata change detected - useful for development", ) @click.option( "--cors", is_flag=True, help="Enable CORS by serving Access-Control-Allow-Origin: *" diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index f0dab3ea..8f770afb 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -14,7 +14,7 @@ Options: -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to automatically assign an available port. - --reload Automatically reload if database or code change detected - + --reload Automatically reload if code or metadata change detected - useful for development --cors Enable CORS by serving Access-Control-Allow-Origin: * From 59ef4a20cba1533bc347378415f4ffcd025f32c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 2 Apr 2021 13:27:03 -0700 Subject: [PATCH 0310/1866] =?UTF-8?q?=C2=A9=202017-2021?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index dd0f7c62..89009ea9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ master_doc = "index" # General information about the project. project = "Datasette" -copyright = "2017-2020, Simon Willison" +copyright = "2017-2021, Simon Willison" author = "Simon Willison" # Disable -- turning into – From 0a7621f96f8ad14da17e7172e8a7bce24ef78966 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 2 Apr 2021 20:42:28 -0700 Subject: [PATCH 0311/1866] Use pytest-xdist to speed up tests (#1290) * Run tests in CI using pytest-xdist * Documentation for pytest-xdist Closes #1289 --- .github/workflows/test.yml | 3 ++- docs/contributing.rst | 18 ++++++++++++++++++ pytest.ini | 2 ++ setup.py | 1 + tests/test_cli_serve_server.py | 3 +++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1774213..bcb241d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,4 +26,5 @@ jobs: pip install -e '.[test]' - name: Run tests run: | - pytest + pytest -n auto -m "not serial" + pytest -m "serial" diff --git a/docs/contributing.rst b/docs/contributing.rst index 7e16280b..c3d0989a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -43,10 +43,28 @@ The next step is to create a virtual environment for your project and use it to That last line does most of the work: ``pip install -e`` means "install this package in a way that allows me to edit the source code in place". The ``.[test]`` option means "use the setup.py in this directory and install the optional testing dependencies as well". +.. _contributing_running_tests: + +Running the tests +----------------- + Once you have done this, you can run the Datasette unit tests from inside your ``datasette/`` directory using `pytest `__ like so:: pytest +You can run the tests faster using multiple CPU cores with `pytest-xdist `__ like this:: + + pytest -n auto -m "not serial" + +``-n auto`` detects the number of available cores automatically. The ``-m "not serial"`` skips tests that don't work well in a parallel test environment. You can run those tests separately like so:: + + pytest -m "serial" + +.. _contributing_using_fixtures: + +Using fixtures +-------------- + To run Datasette itself, type ``datasette``. You're going to need at least one SQLite database. A quick way to get started is to use the fixtures database that Datasette uses for its own tests. diff --git a/pytest.ini b/pytest.ini index aa292efc..d702ce5f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,5 @@ filterwarnings= ignore:Using or importing the ABCs::bs4.element # Python 3.7 PendingDeprecationWarning: Task.current_task() ignore:.*current_task.*:PendingDeprecationWarning +markers = + serial: tests to avoid using with pytest-xdist diff --git a/setup.py b/setup.py index 3540e30a..c67aa6a3 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ setup( "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ "pytest>=5.2.2,<6.3.0", + "pytest-xdist>=2.2.1,<2.3", "pytest-asyncio>=0.10,<0.15", "beautifulsoup4>=4.8.1,<4.10.0", "black==20.8b1", diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 6962d2fd..6f5366d1 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,6 +1,8 @@ import httpx +import pytest +@pytest.mark.serial def test_serve_localhost_http(ds_localhost_http_server): response = httpx.get("http://localhost:8041/_memory.json") assert { @@ -10,6 +12,7 @@ def test_serve_localhost_http(ds_localhost_http_server): }.items() <= response.json().items() +@pytest.mark.serial def test_serve_localhost_https(ds_localhost_https_server): _, client_cert = ds_localhost_https_server response = httpx.get("https://localhost:8042/_memory.json", verify=client_cert) From 6ed9238178a56da5fb019f37fb1e1e15886be1d1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Apr 2021 11:18:17 -0700 Subject: [PATCH 0312/1866] Update pytest-asyncio requirement from <0.15,>=0.10 to >=0.10,<0.16 (#1303) Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.15.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c67aa6a3..03457261 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( "test": [ "pytest>=5.2.2,<6.3.0", "pytest-xdist>=2.2.1,<2.3", - "pytest-asyncio>=0.10,<0.15", + "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", "black==20.8b1", "pytest-timeout>=1.4.2,<1.5", From a4bb2abce0764d49d255e5379f9e9c70981834ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Apr 2021 23:07:37 -0700 Subject: [PATCH 0313/1866] Show primary key cells in bold without affecting columns called 'link', closes #1308 --- datasette/static/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 4c41ea98..617bd2b1 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -432,7 +432,7 @@ td { vertical-align: top; white-space: pre-wrap; } -td.col-link { +td.type-pk { font-weight: bold; } td em { From 5e60bad40460f68122006ce704cfc163d6076f34 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 08:47:21 -0700 Subject: [PATCH 0314/1866] Upgrade to GitHub-native Dependabot (#1314) Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..b969c4c1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 + ignore: + - dependency-name: black + versions: + - 21.4b0 + - 21.4b1 From 1b697539f5b53cec3fe13c0f4ada13ba655c88c7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 08:47:49 -0700 Subject: [PATCH 0315/1866] Bump black from 20.8b1 to 21.4b2 (#1313) Bumps [black](https://github.com/psf/black) from 20.8b1 to 21.4b2. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/master/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 03457261..6f3d9a1c 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup( "pytest-xdist>=2.2.1,<2.3", "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", - "black==20.8b1", + "black==21.4b2", "pytest-timeout>=1.4.2,<1.5", "trustme>=0.7,<0.8", ], From 9b3b7e280ca718254b4ca15d40864297146a85b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 10:19:40 -0700 Subject: [PATCH 0316/1866] Update jinja2 requirement from <2.12.0,>=2.10.3 to >=2.10.3,<3.1.0 (#1324) Updates the requirements on [jinja2](https://github.com/pallets/jinja) to permit the latest version. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.10.3...3.0.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6f3d9a1c..124ce29d 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ setup( "asgiref>=3.2.10,<3.4.0", "click~=7.1.1", "click-default-group~=1.2.2", - "Jinja2>=2.10.3,<2.12.0", + "Jinja2>=2.10.3,<3.1.0", "hupper~=1.9", "httpx>=0.17", "pint~=0.9", From 459259175eddeed727fd8f08dc19a332779a4f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abdussamet=20Ko=C3=A7ak?= Date: Sun, 23 May 2021 02:53:34 +0300 Subject: [PATCH 0317/1866] Fix small typo (#1335) --- docs/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings.rst b/docs/settings.rst index b4c8a50e..af8e4406 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -261,7 +261,7 @@ database file to the URL path for every table and query within that database. When combined with far-future expire headers this ensures that queries can be cached forever, safe in the knowledge that any modifications to the database -itself will result in new, uncachcacheed URL paths. +itself will result in new, uncached URL paths. :: From 593d3e8173b45e20ff3c95afb3df7ceb85bf7fef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 May 2021 16:53:56 -0700 Subject: [PATCH 0318/1866] Update aiofiles requirement from <0.7,>=0.4 to >=0.4,<0.8 (#1330) Updates the requirements on [aiofiles](https://github.com/Tinche/aiofiles) to permit the latest version. - [Release notes](https://github.com/Tinche/aiofiles/releases) - [Commits](https://github.com/Tinche/aiofiles/compare/v0.4.0...v0.7.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 124ce29d..c98cb012 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ setup( "pint~=0.9", "pluggy~=0.13.0", "uvicorn~=0.11", - "aiofiles>=0.4,<0.7", + "aiofiles>=0.4,<0.8", "janus>=0.4,<0.7", "asgi-csrf>=0.6", "PyYAML~=5.3", From b64d87204612a84663616e075f542499a5d82a03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 May 2021 16:54:24 -0700 Subject: [PATCH 0319/1866] Update itsdangerous requirement from ~=1.1 to >=1.1,<3.0 (#1325) Updates the requirements on [itsdangerous](https://github.com/pallets/itsdangerous) to permit the latest version. - [Release notes](https://github.com/pallets/itsdangerous/releases) - [Changelog](https://github.com/pallets/itsdangerous/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/itsdangerous/compare/1.1.0...2.0.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c98cb012..6072044f 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( "asgi-csrf>=0.6", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", - "itsdangerous~=1.1", + "itsdangerous>=1.1,<3.0", "python-baseconv==1.2.2", ], entry_points=""" From 5c3b3ef97eed55895cf48d4a9ee0635c1c4d03b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 May 2021 16:54:48 -0700 Subject: [PATCH 0320/1866] Update click requirement from ~=7.1.1 to >=7.1.1,<8.1.0 (#1323) Updates the requirements on [click](https://github.com/pallets/click) to permit the latest version. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/7.1.1...8.0.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6072044f..b9f8dd7b 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( python_requires=">=3.6", install_requires=[ "asgiref>=3.2.10,<3.4.0", - "click~=7.1.1", + "click>=7.1.1,<8.1.0", "click-default-group~=1.2.2", "Jinja2>=2.10.3,<3.1.0", "hupper~=1.9", From 5e9672c9bb33e41686472db4aa427168f9e67dbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 May 2021 16:55:39 -0700 Subject: [PATCH 0321/1866] Bump black from 21.4b2 to 21.5b1 (#1321) Bumps [black](https://github.com/psf/black) from 21.4b2 to 21.5b1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9f8dd7b..60a94a5e 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup( "pytest-xdist>=2.2.1,<2.3", "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", - "black==21.4b2", + "black==21.5b1", "pytest-timeout>=1.4.2,<1.5", "trustme>=0.7,<0.8", ], From 9789b94da48183dabf105c6419bdcde2634b36a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 22 May 2021 17:34:33 -0700 Subject: [PATCH 0322/1866] ?_facet_size=100 parameter, closes #1332 --- datasette/facets.py | 16 +++++++++---- docs/facets.rst | 2 ++ docs/json_api.rst | 6 +++++ docs/plugin_hooks.rst | 1 + tests/test_facets.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 4 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index 01628760..ff6396d7 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -101,6 +101,14 @@ class Facet: # [('_foo', 'bar'), ('_foo', '2'), ('empty', '')] return urllib.parse.parse_qsl(self.request.query_string, keep_blank_values=True) + def get_facet_size(self): + facet_size = self.ds.setting("default_facet_size") + max_returned_rows = self.ds.setting("max_returned_rows") + custom_facet_size = self.request.args.get("_facet_size") + if custom_facet_size and custom_facet_size.isdigit(): + facet_size = int(custom_facet_size) + return min(facet_size, max_returned_rows) + async def suggest(self): return [] @@ -136,7 +144,7 @@ class ColumnFacet(Facet): async def suggest(self): row_count = await self.get_row_count() columns = await self.get_columns(self.sql, self.params) - facet_size = self.ds.setting("default_facet_size") + facet_size = self.get_facet_size() suggested_facets = [] already_enabled = [c["config"]["simple"] for c in self.get_configs()] for column in columns: @@ -186,7 +194,7 @@ class ColumnFacet(Facet): qs_pairs = self.get_querystring_pairs() - facet_size = self.ds.setting("default_facet_size") + facet_size = self.get_facet_size() for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] @@ -338,7 +346,7 @@ class ArrayFacet(Facet): facet_results = {} facets_timed_out = [] - facet_size = self.ds.setting("default_facet_size") + facet_size = self.get_facet_size() for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] @@ -449,7 +457,7 @@ class DateFacet(Facet): facet_results = {} facets_timed_out = [] args = dict(self.get_querystring_pairs()) - facet_size = self.ds.setting("default_facet_size") + facet_size = self.get_facet_size() for source_and_config in self.get_configs(): config = source_and_config["config"] source = source_and_config["source"] diff --git a/docs/facets.rst b/docs/facets.rst index 3f2f6879..5061d11c 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -84,6 +84,8 @@ This works for both the HTML interface and the ``.json`` view. When enabled, fac If Datasette detects that a column is a foreign key, the ``"label"`` property will be automatically derived from the detected label column on the referenced table. +The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). + Facets in metadata.json ----------------------- diff --git a/docs/json_api.rst b/docs/json_api.rst index 0f88cb07..9efacf35 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -371,6 +371,12 @@ Special table arguments Pagination by continuation token - pass the token that was returned in the ``"next"`` property by the previous page. +``?_facet=column`` + Facet by column. Can be applied multiple times, see :ref:`facets`. Only works on the default JSON output, not on any of the custom shapes. + +``?_facet_size=100`` + Increase the number of facet results returned for each facet. + ``?_trace=1`` Turns on tracing for this page: SQL queries executed during the request will be gathered and included in the response, either in a new ``"_traces"`` key diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 0a176add..7a1645ec 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -619,6 +619,7 @@ Each Facet subclass implements a new type of facet operation. The class should l # using self.sql and self.params as the starting point facet_results = {} facets_timed_out = [] + facet_size = self.get_facet_size() # Do some calculations here... for column in columns_selected_for_facet: try: diff --git a/tests/test_facets.py b/tests/test_facets.py index 31518682..a1a14e71 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -347,3 +347,55 @@ async def test_json_array_with_blanks_and_nulls(): "toggle_url": "http://localhost/test_json_array/foo.json?_facet_array=json_column", } ] + + +@pytest.mark.asyncio +async def test_facet_size(): + ds = Datasette([], memory=True, config={"max_returned_rows": 50}) + db = ds.add_database(Database(ds, memory_name="test_facet_size")) + await db.execute_write( + "create table neighbourhoods(city text, neighbourhood text)", block=True + ) + for i in range(1, 51): + for j in range(1, 4): + await db.execute_write( + "insert into neighbourhoods (city, neighbourhood) values (?, ?)", + ["City {}".format(i), "Neighbourhood {}".format(j)], + block=True, + ) + response = await ds.client.get("/test_facet_size/neighbourhoods.json") + data = response.json() + assert data["suggested_facets"] == [ + { + "name": "neighbourhood", + "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet=neighbourhood", + } + ] + # Bump up _facet_size= to suggest city too + response2 = await ds.client.get( + "/test_facet_size/neighbourhoods.json?_facet_size=50" + ) + data2 = response2.json() + assert sorted(data2["suggested_facets"], key=lambda f: f["name"]) == [ + { + "name": "city", + "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city", + }, + { + "name": "neighbourhood", + "toggle_url": "http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=neighbourhood", + }, + ] + # Facet by city should return expected number of results + response3 = await ds.client.get( + "/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city" + ) + data3 = response3.json() + assert len(data3["facet_results"]["city"]["results"]) == 50 + # Reduce max_returned_rows and check that it's respected + ds._settings["max_returned_rows"] = 20 + response4 = await ds.client.get( + "/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city" + ) + data4 = response4.json() + assert len(data4["facet_results"]["city"]["results"]) == 20 From a443dba82f43c22b03402a4f86c85558ccb526b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 22 May 2021 17:45:54 -0700 Subject: [PATCH 0323/1866] Release 0.57a0 Refs #1281, #1282, #1289, #1290, #1308, #1313, #1314, #1321, #1323, #1325, #1330, #1332, #1335 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 4dcf73b0..4da56e0a 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.56" +__version__ = "0.57a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 756badce..e00791f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_57_a0: + +0.57a0 (2021-05-22) +------------------- + +Mainly dependency bumps, plus a new ``?_facet_size=`` argument. + +- Updated dependencies: pytest-asyncio, Black, jinja2, aiofiles, itsdangerous +- Fixed bug where columns called "Link" were incorrectly displayed in bold. (:issue:`1308`) +- New ``?_facet_size=`` argument for customizing the number of facet results returned on a page. (:issue:`1332`) + .. _v0_56: 0.56 (2021-03-28) From 2bd9d54b2762c991e11950c22c88c0336158d49b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 23 May 2021 18:41:50 -0700 Subject: [PATCH 0324/1866] Fix Jinja warnings, closes #1338, refs #1331 --- datasette/app.py | 5 ++--- datasette/views/database.py | 10 +++++----- datasette/views/table.py | 24 +++++++++++++----------- docs/plugin_hooks.rst | 8 ++++---- tests/plugins/my_plugin_2.py | 8 ++++---- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ee816426..e284995a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -19,9 +19,8 @@ import urllib.parse from concurrent import futures from pathlib import Path -from markupsafe import Markup +from markupsafe import Markup, escape from itsdangerous import URLSafeSerializer -import jinja2 from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound @@ -864,7 +863,7 @@ class Datasette: } if request and request.args.get("_context") and self.setting("template_debug"): return "
    {}
    ".format( - jinja2.escape(json.dumps(template_context, default=repr, indent=4)) + escape(json.dumps(template_context, default=repr, indent=4)) ) return await template.render_async(template_context) diff --git a/datasette/views/database.py b/datasette/views/database.py index 0c58a351..96b2ca91 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,8 +1,8 @@ import os import hashlib import itertools -import jinja2 import json +from markupsafe import Markup, escape from urllib.parse import parse_qsl, urlencode from datasette.utils import ( @@ -354,11 +354,11 @@ class QueryView(DataView): display_value = plugin_value else: if value in ("", None): - display_value = jinja2.Markup(" ") + display_value = Markup(" ") elif is_url(str(display_value).strip()): - display_value = jinja2.Markup( + display_value = Markup( '{url}'.format( - url=jinja2.escape(value.strip()) + url=escape(value.strip()) ) ) elif isinstance(display_value, bytes): @@ -372,7 +372,7 @@ class QueryView(DataView): ).hexdigest(), }, ) - display_value = jinja2.Markup( + display_value = Markup( '<Binary: {} byte{}>'.format( blob_url, len(display_value), diff --git a/datasette/views/table.py b/datasette/views/table.py index 48792284..8007377a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -2,7 +2,7 @@ import urllib import itertools import json -import jinja2 +import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted @@ -135,12 +135,12 @@ class RowTableShared(DataView): "value_type": "pk", "is_special_link_column": is_special_link_column, "raw": pk_path, - "value": jinja2.Markup( + "value": markupsafe.Markup( '{flat_pks}'.format( base_url=base_url, database=database, table=urllib.parse.quote_plus(table), - flat_pks=str(jinja2.escape(pk_path)), + flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) ), @@ -166,7 +166,7 @@ class RowTableShared(DataView): if plugin_display_value is not None: display_value = plugin_display_value elif isinstance(value, bytes): - display_value = jinja2.Markup( + display_value = markupsafe.Markup( '<Binary: {} byte{}>'.format( self.ds.urls.row_blob( database, @@ -187,22 +187,22 @@ class RowTableShared(DataView): link_template = ( LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE ) - display_value = jinja2.Markup( + display_value = markupsafe.Markup( link_template.format( database=database, base_url=base_url, table=urllib.parse.quote_plus(other_table), link_id=urllib.parse.quote_plus(str(value)), - id=str(jinja2.escape(value)), - label=str(jinja2.escape(label)) or "-", + id=str(markupsafe.escape(value)), + label=str(markupsafe.escape(label)) or "-", ) ) elif value in ("", None): - display_value = jinja2.Markup(" ") + display_value = markupsafe.Markup(" ") elif is_url(str(value).strip()): - display_value = jinja2.Markup( + display_value = markupsafe.Markup( '{url}'.format( - url=jinja2.escape(value.strip()) + url=markupsafe.escape(value.strip()) ) ) elif column in table_metadata.get("units", {}) and value != "": @@ -212,7 +212,9 @@ class RowTableShared(DataView): # representation, which we have to round off to avoid ugliness. In the vast # majority of cases this rounding will be inconsequential. I hope. value = round(value.to_compact(), 6) - display_value = jinja2.Markup(f"{value:~P}".replace(" ", " ")) + display_value = markupsafe.Markup( + f"{value:~P}".replace(" ", " ") + ) else: display_value = str(value) if truncate_cells and len(display_value) > truncate_cells: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 7a1645ec..688eaa61 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -389,7 +389,7 @@ If the value matches that pattern, the plugin returns an HTML link element: .. code-block:: python from datasette import hookimpl - import jinja2 + import markupsafe import json @@ -415,9 +415,9 @@ If the value matches that pattern, the plugin returns an HTML link element: or href.startswith("https://") ): return None - return jinja2.Markup('{label}'.format( - href=jinja2.escape(data["href"]), - label=jinja2.escape(data["label"] or "") or " " + return markupsafe.Markup('{label}'.format( + href=markupsafe.escape(data["href"]), + label=markupsafe.escape(data["label"] or "") or " " )) Examples: `datasette-render-binary `_, `datasette-render-markdown `__, `datasette-json-html `__ diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 6cd222e6..f3b794cf 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -1,6 +1,6 @@ from datasette import hookimpl from functools import wraps -import jinja2 +import markupsafe import json @@ -38,11 +38,11 @@ def render_cell(value, database): or href.startswith("https://") ): return None - return jinja2.Markup( + return markupsafe.Markup( '{label}'.format( database=database, - href=jinja2.escape(data["href"]), - label=jinja2.escape(data["label"] or "") or " ", + href=markupsafe.escape(data["href"]), + label=markupsafe.escape(data["label"] or "") or " ", ) ) From eae3084b46e2c3931db12cdef79093ad0e644bce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 May 2021 10:52:09 -0700 Subject: [PATCH 0325/1866] Fixed another Jinja warning, refs #1338 --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index e284995a..957ced7c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -21,7 +21,7 @@ from pathlib import Path from markupsafe import Markup, escape from itsdangerous import URLSafeSerializer -from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape +from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound import uvicorn From fc972350a8a0276d87a6a83efbbdfab0edd060d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 May 2021 11:07:03 -0700 Subject: [PATCH 0326/1866] Docker image should now allow apt-get install, closes #1320 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8193700d..7c56cf56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && \ apt-get remove -y software-properties-common && \ apt clean && \ rm -rf /var/lib/apt && \ - rm -rf /var/lib/dpkg + rm -rf /var/lib/dpkg/info/* RUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \ find /usr/local/lib -name '__pycache__' | xargs rm -r && \ From 56af118fc158a59a98688f2caa6f01db6b68da83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 May 2021 11:14:45 -0700 Subject: [PATCH 0327/1866] How to apt-get install in Docker container, refs #1320 --- docs/installation.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 6ac67f59..381d9a63 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -195,3 +195,12 @@ You can now run the new custom image like so:: You can confirm that the plugins are installed by visiting http://127.0.0.1:8001/-/plugins + +Some plugins such as `datasette-ripgrep `__ may need additional system packages. You can install these by running `apt-get install` inside the container: + + docker run datasette-057a0 bash -c ' + apt-get update && + apt-get install ripgrep && + pip install datasette-ripgrep' + + docker commit $(docker ps -lq) datasette-with-ripgrep From c0a748e5c3f498fa8c139b420d07dd3dea612379 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 24 May 2021 11:15:15 -0700 Subject: [PATCH 0328/1866] Markup fix, refs #1320 --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 381d9a63..b6881bc0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -196,7 +196,7 @@ You can now run the new custom image like so:: You can confirm that the plugins are installed by visiting http://127.0.0.1:8001/-/plugins -Some plugins such as `datasette-ripgrep `__ may need additional system packages. You can install these by running `apt-get install` inside the container: +Some plugins such as `datasette-ripgrep `__ may need additional system packages. You can install these by running `apt-get install` inside the container:: docker run datasette-057a0 bash -c ' apt-get update && From f1c29fd6a184254aa68efadf096bcf21e848f921 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 May 2021 21:17:43 -0700 Subject: [PATCH 0329/1866] ?_col=/?_nocol= to show/hide columns on the table page Closes #615 * Cog icon for hiding columns * Show all columns cog menu item * Do not allow hide column on primary keys * Allow both ?_col= and ?_nocol= * De-duplicate if ?_col= passed multiple times * 400 error if user tries to ?_nocol= a primary key * Documentation for ?_col= and ?_nocol= --- datasette/static/table.js | 45 +++++++++++++++++++++++------ datasette/views/table.py | 47 +++++++++++++++++++++++++++---- docs/json_api.rst | 6 ++++ tests/test_api.py | 59 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 15 deletions(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index b4e1e113..4c24d772 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -4,6 +4,8 @@ var DROPDOWN_HTML = `
  • Sort ascending
  • Sort descending
  • Facet by this
    • +
  • Hide this column
    • +
  • Show all columns
  • Show not-blank rows

    • @@ -24,7 +26,7 @@ var DROPDOWN_ICON_SVG = ` el.innerText.trim() == "").length ) { - notBlank.style.display = "block"; + notBlank.parentNode.style.display = "block"; notBlank.setAttribute("href", notBlankUrl(column)); } else { - notBlank.style.display = "none"; + notBlank.parentNode.style.display = "none"; } var columnTypeP = menu.querySelector(".dropdown-column-type"); var columnType = th.dataset.columnType; diff --git a/datasette/views/table.py b/datasette/views/table.py index 8007377a..b54a908a 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -64,6 +64,41 @@ class Row: class RowTableShared(DataView): + async def columns_to_select(self, db, table, request): + table_columns = await db.table_columns(table) + pks = await db.primary_keys(table) + columns = list(table_columns) + if "_col" in request.args: + columns = list(pks) + _cols = request.args.getlist("_col") + bad_columns = [column for column in _cols if column not in table_columns] + if bad_columns: + raise DatasetteError( + "_col={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + # De-duplicate maintaining order: + columns.extend(dict.fromkeys(_cols)) + if "_nocol" in request.args: + # Return all columns EXCEPT these + bad_columns = [ + column + for column in request.args.getlist("_nocol") + if (column not in table_columns) or (column in pks) + ] + if bad_columns: + raise DatasetteError( + "_nocol={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + tmp_columns = [ + column + for column in columns + if column not in request.args.getlist("_nocol") + ] + columns = tmp_columns + return columns + async def sortable_columns_for_table(self, database, table, use_rowid): db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) @@ -323,18 +358,16 @@ class TableView(RowTableShared): ) pks = await db.primary_keys(table) - table_column_details = await db.table_column_details(table) - table_columns = [column.name for column in table_column_details] - - select_columns = ", ".join(escape_sqlite(t) for t in table_columns) + table_columns = await self.columns_to_select(db, table, request) + select_clause = ", ".join(escape_sqlite(t) for t in table_columns) use_rowid = not pks and not is_view if use_rowid: - select = f"rowid, {select_columns}" + select = f"rowid, {select_clause}" order_by = "rowid" order_by_pks = "rowid" else: - select = select_columns + select = select_clause order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) order_by = order_by_pks @@ -717,6 +750,8 @@ class TableView(RowTableShared): column = fk["column"] if column not in columns_to_expand: continue + if column not in columns: + continue expanded_columns.append(column) # Gather the values column_index = columns.index(column) diff --git a/docs/json_api.rst b/docs/json_api.rst index 9efacf35..787b1203 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -296,6 +296,12 @@ You can filter the data returned by the table based on column values using a que Special table arguments ~~~~~~~~~~~~~~~~~~~~~~~ +``?_col=COLUMN1&_col=COLUMN2`` + List specific columns to display. These will be shown along with any primary keys. + +``?_nocol=COLUMN1&_nocol=COLUMN2`` + List specific columns to hide - any column not listed will be displayed. Primary keys cannot be hidden. + ``?_labels=on/off`` Expand foreign key references for every possible column. See below. diff --git a/tests/test_api.py b/tests/test_api.py index caf23329..00de84e6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2009,3 +2009,62 @@ def test_http_options_request(app_client): response = app_client.request("/fixtures", method="OPTIONS") assert response.status == 200 assert response.text == "ok" + + +@pytest.mark.parametrize( + "path,expected_columns", + ( + ("/fixtures/facetable.json?_col=created", ["pk", "created"]), + ( + "/fixtures/facetable.json?_nocol=created", + [ + "pk", + "planet_int", + "on_earth", + "state", + "city_id", + "neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + ), + ( + "/fixtures/facetable.json?_col=state&_col=created", + ["pk", "state", "created"], + ), + ( + "/fixtures/facetable.json?_col=state&_col=state", + ["pk", "state"], + ), + ( + "/fixtures/facetable.json?_col=state&_col=created&_nocol=created", + ["pk", "state"], + ), + ( + "/fixtures/simple_view.json?_nocol=content", + ["upper_content"], + ), + ("/fixtures/simple_view.json?_col=content", ["content"]), + ), +) +def test_col_nocol(app_client, path, expected_columns): + response = app_client.get(path) + assert response.status == 200 + columns = response.json["columns"] + assert columns == expected_columns + + +@pytest.mark.parametrize( + "path,expected_error", + ( + ("/fixtures/facetable.json?_col=bad", "_col=bad - invalid columns"), + ("/fixtures/facetable.json?_nocol=bad", "_nocol=bad - invalid columns"), + ("/fixtures/facetable.json?_nocol=pk", "_nocol=pk - invalid columns"), + ("/fixtures/simple_view.json?_col=bad", "_col=bad - invalid columns"), + ), +) +def test_col_nocol_errors(app_client, path, expected_error): + response = app_client.get(path) + assert response.status == 400 + assert response.json["error"] == expected_error From 51d788114035458d8f860d9ea6d74078e6c0ea0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 May 2021 21:31:12 -0700 Subject: [PATCH 0330/1866] 'Show all columns' menu item if any _col= set, closes #1341 Refs #615 --- datasette/static/table.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index 4c24d772..991346df 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -55,6 +55,7 @@ var DROPDOWN_ICON_SVG = ` Date: Thu, 27 May 2021 09:00:58 -0700 Subject: [PATCH 0331/1866] ?_facet_size=max, ... now links to that, closes #1337 Refs #1332 --- datasette/facets.py | 4 ++- datasette/static/app.css | 9 ++++- datasette/templates/table.html | 4 ++- tests/fixtures.py | 20 +++++------ tests/test_html.py | 62 ++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index ff6396d7..9d95d0f3 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -105,7 +105,9 @@ class Facet: facet_size = self.ds.setting("default_facet_size") max_returned_rows = self.ds.setting("max_returned_rows") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size and custom_facet_size.isdigit(): + if custom_facet_size == "max": + facet_size = max_returned_rows + elif custom_facet_size and custom_facet_size.isdigit(): facet_size = int(custom_facet_size) return min(facet_size, max_returned_rows) diff --git a/datasette/static/app.css b/datasette/static/app.css index 617bd2b1..7f04a162 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -645,13 +645,20 @@ form button[type=button] { .facet-info a.cross:active { text-decoration: none; } +ul li.facet-truncated { + list-style-type: none; + position: relative; + top: -0.35em; + text-indent: 0.85em; +} + .advanced-export { margin-top: 1em; padding: 0.01em 2em 0.01em 1em; width: auto; display: inline-block; box-shadow: 1px 2px 8px 2px rgba(0,0,0,0.08); - background-color: white; + background-color: white; } .download-sqlite em { diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 077332dc..211352b5 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -162,7 +162,9 @@ {% endif %} {% endfor %} {% if facet_info.truncated %} -
  • ...
    • +
  • {% if request.args._facet_size != "max" -%} + …{% else -%}…{% endif %} +
    • {% endif %}
    diff --git a/tests/fixtures.py b/tests/fixtures.py index 0a721d3a..5730c1bf 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -129,16 +129,16 @@ def make_app_client( files.append(extra_filepath) os.chdir(os.path.dirname(filepath)) config = config or {} - config.update( - { - "default_page_size": 50, - "max_returned_rows": max_returned_rows or 100, - "sql_time_limit_ms": sql_time_limit_ms or 200, - # Default is 3 but this results in "too many open files" - # errors when running the full test suite: - "num_sql_threads": 1, - } - ) + for key, value in { + "default_page_size": 50, + "max_returned_rows": max_returned_rows or 100, + "sql_time_limit_ms": sql_time_limit_ms or 200, + # Default is 3 but this results in "too many open files" + # errors when running the full test suite: + "num_sql_threads": 1, + }.items(): + if key not in config: + config[key] = value ds = Datasette( files, immutables=immutables, diff --git a/tests/test_html.py b/tests/test_html.py index 9e86ebc2..4f2cc8ad 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1612,3 +1612,65 @@ def test_navigation_menu_links( assert ( details.find("a", {"href": link}) is None ), f"{link} found but should not have been in nav menu" + + +@pytest.mark.parametrize( + "max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url", + ( + ( + 5, + # Default should show 2 facets + "/fixtures/facetable?_facet=neighborhood", + 2, + True, + "/fixtures/facetable?_facet=neighborhood&_facet_size=max", + ), + # _facet_size above max_returned_rows should show max_returned_rows (5) + ( + 5, + "/fixtures/facetable?_facet=neighborhood&_facet_size=50", + 5, + True, + "/fixtures/facetable?_facet=neighborhood&_facet_size=max", + ), + # If max_returned_rows is high enough, should return all + ( + 20, + "/fixtures/facetable?_facet=neighborhood&_facet_size=max", + 14, + False, + None, + ), + # If num facets > max_returned_rows, show ... without a link + # _facet_size above max_returned_rows should show max_returned_rows (5) + ( + 5, + "/fixtures/facetable?_facet=neighborhood&_facet_size=max", + 5, + True, + None, + ), + ), +) +def test_facet_more_links( + max_returned_rows, + path, + expected_num_facets, + expected_ellipses, + expected_ellipses_url, +): + with make_app_client( + config={"max_returned_rows": max_returned_rows, "default_facet_size": 2} + ) as client: + response = client.get(path) + soup = Soup(response.body, "html.parser") + lis = soup.select("#facet-neighborhood ul li:not(.facet-truncated)") + facet_truncated = soup.select_one(".facet-truncated") + assert len(lis) == expected_num_facets + if not expected_ellipses: + assert facet_truncated is None + else: + if expected_ellipses_url: + assert facet_truncated.find("a")["href"] == expected_ellipses_url + else: + assert facet_truncated.find("a") is None From 4545120c920165aad9659d27111f63f977b8a399 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 May 2021 09:04:26 -0700 Subject: [PATCH 0332/1866] Test and docs for ?_facet_size=max, refs #1337 --- docs/json_api.rst | 2 +- tests/test_facets.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index 787b1203..e48ec514 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -381,7 +381,7 @@ Special table arguments Facet by column. Can be applied multiple times, see :ref:`facets`. Only works on the default JSON output, not on any of the custom shapes. ``?_facet_size=100`` - Increase the number of facet results returned for each facet. + Increase the number of facet results returned for each facet. Use ``?_facet_size=max`` for the maximum available size, determined by :ref:`setting_max_returned_rows`. ``?_trace=1`` Turns on tracing for this page: SQL queries executed during the request will diff --git a/tests/test_facets.py b/tests/test_facets.py index a1a14e71..18fb8c3b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -399,3 +399,9 @@ async def test_facet_size(): ) data4 = response4.json() assert len(data4["facet_results"]["city"]["results"]) == 20 + # Test _facet_size=max + response5 = await ds.client.get( + "/test_facet_size/neighbourhoods.json?_facet_size=max&_facet=city" + ) + data5 = response5.json() + assert len(data5["facet_results"]["city"]["results"]) == 20 From 1a8972f9c012cd22b088c6b70661a9c3d3847853 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 May 2021 09:11:03 -0700 Subject: [PATCH 0333/1866] Upgrade Heroku runtime to python-3.8.10 --- datasette/publish/heroku.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/publish/heroku.py b/datasette/publish/heroku.py index 19fe3fbe..2ebbd4bd 100644 --- a/datasette/publish/heroku.py +++ b/datasette/publish/heroku.py @@ -175,7 +175,7 @@ def temporary_heroku_directory( fp.write(json.dumps(metadata_content, indent=2)) with open("runtime.txt", "w") as fp: - fp.write("python-3.8.7") + fp.write("python-3.8.10") if branch: install = [ From 89822d10be0da446471986addea91d9766f12efb Mon Sep 17 00:00:00 2001 From: Blair Drummond <10801138+blairdrummond@users.noreply.github.com> Date: Thu, 27 May 2021 12:49:23 -0400 Subject: [PATCH 0334/1866] Docker multi-arch support with Buildx (#1319) Thanks, @blairdrummond --- .github/workflows/push_docker_tag.yml | 34 ++++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index 9a3969f0..e61150a5 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -11,15 +11,31 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Available platforms + run: echo ${{ steps.buildx.outputs.platforms }} + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + - name: Build and push to Docker Hub + run: | + docker buildx build \ + --file Dockerfile . \ + --tag $REPO:${VERSION_TAG} \ + --build-arg VERSION=${VERSION_TAG} \ + --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ + --push env: - DOCKER_USER: ${{ secrets.DOCKER_USER }} - DOCKER_PASS: ${{ secrets.DOCKER_PASS }} + REPO: datasetteproject/datasette VERSION_TAG: ${{ github.event.inputs.version_tag }} - run: |- - docker login -u $DOCKER_USER -p $DOCKER_PASS - export REPO=datasetteproject/datasette - docker build -f Dockerfile \ - -t $REPO:${VERSION_TAG} \ - --build-arg VERSION=${VERSION_TAG} . - docker push $REPO:${VERSION_TAG} From 7b106e106000713bbee31b34d694b3dadbd4818c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 May 2021 09:54:21 -0700 Subject: [PATCH 0335/1866] Release 0.57a1 Refs #1319, #1320, #1331, #1337, #1338, #1341 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 4da56e0a..cc98e271 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.57a0" +__version__ = "0.57a1" __version_info__ = tuple(__version__.split(".")) From f7d3e76fb3d1fa5aabe339251e4a930610643822 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 May 2021 22:31:14 -0400 Subject: [PATCH 0336/1866] Facets now execute ignoring ?_col and ?_nocol, fixes #1345 --- datasette/views/table.py | 30 ++++++++++++++++++++++-------- tests/test_api.py | 15 +++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index b54a908a..c5703292 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -358,16 +358,21 @@ class TableView(RowTableShared): ) pks = await db.primary_keys(table) - table_columns = await self.columns_to_select(db, table, request) - select_clause = ", ".join(escape_sqlite(t) for t in table_columns) + table_columns = await db.table_columns(table) + + specified_columns = await self.columns_to_select(db, table, request) + select_specified_columns = ", ".join( + escape_sqlite(t) for t in specified_columns + ) + select_all_columns = ", ".join(escape_sqlite(t) for t in table_columns) use_rowid = not pks and not is_view if use_rowid: - select = f"rowid, {select_clause}" + select_specified_columns = f"rowid, {select_specified_columns}" + select_all_columns = f"rowid, {select_all_columns}" order_by = "rowid" order_by_pks = "rowid" else: - select = select_clause order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) order_by = order_by_pks @@ -633,7 +638,7 @@ class TableView(RowTableShared): where_clause = f"where {' and '.join(where_clauses)} " if order_by: - order_by = f"order by {order_by} " + order_by = f"order by {order_by}" extra_args = {} # Handle ?_size=500 @@ -656,13 +661,22 @@ class TableView(RowTableShared): else: page_size = self.ds.page_size - sql_no_limit = "select {select} from {table_name} {where}{order_by}".format( - select=select, + sql_no_limit = ( + "select {select_all_columns} from {table_name} {where}{order_by}".format( + select_all_columns=select_all_columns, + table_name=escape_sqlite(table), + where=where_clause, + order_by=order_by, + ) + ) + sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( + select_specified_columns=select_specified_columns, table_name=escape_sqlite(table), where=where_clause, order_by=order_by, + page_size=page_size + 1, + offset=offset, ) - sql = f"{sql_no_limit.rstrip()} limit {page_size + 1}{offset}" if request.args.get("_timelimit"): extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) diff --git a/tests/test_api.py b/tests/test_api.py index 00de84e6..2c5d7516 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2041,6 +2041,21 @@ def test_http_options_request(app_client): "/fixtures/facetable.json?_col=state&_col=created&_nocol=created", ["pk", "state"], ), + ( + # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345 + "/fixtures/facetable.json?_nocol=state&_facet=state", + [ + "pk", + "created", + "planet_int", + "on_earth", + "city_id", + "neighborhood", + "tags", + "complex_array", + "distinct_some_null", + ], + ), ( "/fixtures/simple_view.json?_nocol=content", ["upper_content"], From c5ae1197a208e1b034c88882e3ac865813a40980 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 May 2021 22:39:14 -0400 Subject: [PATCH 0337/1866] ?_nofacets=1 option, closes #1350 --- datasette/views/table.py | 16 +++++++++------- docs/facets.rst | 2 ++ docs/json_api.rst | 3 +++ tests/test_api.py | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index c5703292..83c2b922 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -731,13 +731,14 @@ class TableView(RowTableShared): ) ) - for facet in facet_instances: - ( - instance_facet_results, - instance_facets_timed_out, - ) = await facet.facet_results() - facet_results.update(instance_facet_results) - facets_timed_out.extend(instance_facets_timed_out) + if not request.args.get("_nofacets"): + for facet in facet_instances: + ( + instance_facet_results, + instance_facets_timed_out, + ) = await facet.facet_results() + facet_results.update(instance_facet_results) + facets_timed_out.extend(instance_facets_timed_out) # Figure out columns and rows for the query columns = [r[0] for r in results.description] @@ -828,6 +829,7 @@ class TableView(RowTableShared): self.ds.setting("suggest_facets") and self.ds.setting("allow_facet") and not _next + and not request.args.get("_nofacets") ): for facet in facet_instances: suggested_facets.extend(await facet.suggest()) diff --git a/docs/facets.rst b/docs/facets.rst index 5061d11c..7730e4ac 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -86,6 +86,8 @@ If Datasette detects that a column is a foreign key, the ``"label"`` property wi The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). +.. _facets_metadata: + Facets in metadata.json ----------------------- diff --git a/docs/json_api.rst b/docs/json_api.rst index e48ec514..62c208a2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -383,6 +383,9 @@ Special table arguments ``?_facet_size=100`` Increase the number of facet results returned for each facet. Use ``?_facet_size=max`` for the maximum available size, determined by :ref:`setting_max_returned_rows`. +``?_nofacets=1`` + Disable all facets and facet suggestions for this page, including any defined by :ref:`facets_metadata`. + ``?_trace=1`` Turns on tracing for this page: SQL queries executed during the request will be gathered and included in the response, either in a new ``"_traces"`` key diff --git a/tests/test_api.py b/tests/test_api.py index 2c5d7516..3d6d0330 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1669,6 +1669,20 @@ def test_suggest_facets_off(): assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] +@pytest.mark.parametrize("nofacets", (True, False)) +def test_nofacets(app_client, nofacets): + path = "/fixtures/facetable.json?_facet=state" + if nofacets: + path += "&_nofacets=1" + response = app_client.get(path) + if nofacets: + assert response.json["suggested_facets"] == [] + assert response.json["facet_results"] == {} + else: + assert response.json["suggested_facets"] != [] + assert response.json["facet_results"] != {} + + def test_expand_labels(app_client): response = app_client.get( "/fixtures/facetable.json?_shape=object&_labels=1&_size=2" From d1d06ace49606da790a765689b4fbffa4c6deecb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 08:49:50 -0700 Subject: [PATCH 0338/1866] ?_trac=1 for CSV, plus ?_nofacets=1 when rendering CSV Closes #1351, closes #1350 --- datasette/utils/__init__.py | 9 +++++++++ datasette/views/base.py | 38 +++++++++++++++++++++++++++++++++---- tests/test_csv.py | 24 ++++++++++++++++++++--- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1fedb69c..dd47771f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -7,6 +7,7 @@ import hashlib import inspect import itertools import json +import markupsafe import mergedeep import os import re @@ -777,6 +778,14 @@ class LimitedWriter: await self.writer.write(bytes) +class EscapeHtmlWriter: + def __init__(self, writer): + self.writer = writer + + async def write(self, content): + await self.writer.write(markupsafe.escape(content)) + + _infinities = {float("inf"), float("-inf")} diff --git a/datasette/views/base.py b/datasette/views/base.py index ba0f7d4c..aefaec6c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -13,6 +13,7 @@ from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, + EscapeHtmlWriter, InvalidSql, LimitedWriter, call_with_supported_arguments, @@ -262,6 +263,16 @@ class DataView(BaseView): async def as_csv(self, request, database, hash, **kwargs): stream = request.args.get("_stream") + # Do not calculate facets: + if not request.args.get("_nofacets"): + if not request.query_string: + new_query_string = "_nofacets=1" + else: + new_query_string = request.query_string + "&_nofacets=1" + new_scope = dict( + request.scope, query_string=new_query_string.encode("latin-1") + ) + request.scope = new_scope if stream: # Some quick sanity checks if not self.ds.setting("allow_csv_stream"): @@ -298,9 +309,27 @@ class DataView(BaseView): if column in expanded_columns: headings.append(f"{column}_label") + content_type = "text/plain; charset=utf-8" + preamble = "" + postamble = "" + + trace = request.args.get("_trace") + if trace: + content_type = "text/html; charset=utf-8" + preamble = ( + "CSV debug" + '" + async def stream_fn(r): - nonlocal data - writer = csv.writer(LimitedWriter(r, self.ds.setting("max_csv_mb"))) + nonlocal data, trace + limited_writer = LimitedWriter(r, self.ds.setting("max_csv_mb")) + if trace: + await limited_writer.write(preamble) + writer = csv.writer(EscapeHtmlWriter(limited_writer)) + else: + writer = csv.writer(limited_writer) first = True next = None while first or (next and stream): @@ -371,13 +400,14 @@ class DataView(BaseView): sys.stderr.flush() await r.write(str(e)) return + await limited_writer.write(postamble) - content_type = "text/plain; charset=utf-8" headers = {} if self.ds.cors: headers["Access-Control-Allow-Origin"] = "*" if request.args.get("_dl", None): - content_type = "text/csv; charset=utf-8" + if not trace: + content_type = "text/csv; charset=utf-8" disposition = 'attachment; filename="{}.csv"'.format( kwargs.get("table", database) ) diff --git a/tests/test_csv.py b/tests/test_csv.py index 6b17033c..30afbd9e 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,3 +1,4 @@ +from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, app_client_csv_max_mb_one, @@ -51,7 +52,7 @@ pk,foreign_key_with_label,foreign_key_with_label_label,foreign_key_with_blank_la def test_table_csv(app_client): - response = app_client.get("/fixtures/simple_primary_key.csv") + response = app_client.get("/fixtures/simple_primary_key.csv?_oh=1") assert response.status == 200 assert not response.headers.get("Access-Control-Allow-Origin") assert "text/plain; charset=utf-8" == response.headers["content-type"] @@ -104,8 +105,8 @@ def test_custom_sql_csv_blob_columns(app_client): assert "text/plain; charset=utf-8" == response.headers["content-type"] assert response.text == ( "rowid,data\r\n" - '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' - '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' + '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacets=1&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' + '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacets=1&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' "3,\r\n" ) @@ -157,3 +158,20 @@ def test_table_csv_stream(app_client): # With _stream=1 should return header + 1001 rows response = app_client.get("/fixtures/compound_three_primary_keys.csv?_stream=1") assert 1002 == len([b for b in response.body.split(b"\r\n") if b]) + + +def test_csv_trace(app_client): + response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") + assert response.headers["content-type"] == "text/html; charset=utf-8" + soup = Soup(response.text, "html.parser") + assert ( + soup.find("textarea").text + == "id,content\r\n1,hello\r\n2,world\r\n3,\r\n4,RENDER_CELL_DEMO\r\n" + ) + assert "select id, content from simple_primary_key" in soup.find("pre").text + + +def test_table_csv_stream_does_not_calculate_facets(app_client): + response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") + soup = Soup(response.text, "html.parser") + assert "select content, count(*) as n" not in soup.find("pre").text From 8bde6c54615af529e81de559cbb3bf3ee5fe17cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 08:55:28 -0700 Subject: [PATCH 0339/1866] Rename ?_nofacets=1 to ?_nofacet=1, refs #1353 --- datasette/views/base.py | 6 +++--- datasette/views/table.py | 4 ++-- docs/json_api.rst | 2 +- tests/test_api.py | 10 +++++----- tests/test_csv.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index aefaec6c..b8c581fc 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -264,11 +264,11 @@ class DataView(BaseView): async def as_csv(self, request, database, hash, **kwargs): stream = request.args.get("_stream") # Do not calculate facets: - if not request.args.get("_nofacets"): + if not request.args.get("_nofacet"): if not request.query_string: - new_query_string = "_nofacets=1" + new_query_string = "_nofacet=1" else: - new_query_string = request.query_string + "&_nofacets=1" + new_query_string = request.query_string + "&_nofacet=1" new_scope = dict( request.scope, query_string=new_query_string.encode("latin-1") ) diff --git a/datasette/views/table.py b/datasette/views/table.py index 83c2b922..7fbf670b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -731,7 +731,7 @@ class TableView(RowTableShared): ) ) - if not request.args.get("_nofacets"): + if not request.args.get("_nofacet"): for facet in facet_instances: ( instance_facet_results, @@ -829,7 +829,7 @@ class TableView(RowTableShared): self.ds.setting("suggest_facets") and self.ds.setting("allow_facet") and not _next - and not request.args.get("_nofacets") + and not request.args.get("_nofacet") ): for facet in facet_instances: suggested_facets.extend(await facet.suggest()) diff --git a/docs/json_api.rst b/docs/json_api.rst index 62c208a2..f1c347b7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -383,7 +383,7 @@ Special table arguments ``?_facet_size=100`` Increase the number of facet results returned for each facet. Use ``?_facet_size=max`` for the maximum available size, determined by :ref:`setting_max_returned_rows`. -``?_nofacets=1`` +``?_nofacet=1`` Disable all facets and facet suggestions for this page, including any defined by :ref:`facets_metadata`. ``?_trace=1`` diff --git a/tests/test_api.py b/tests/test_api.py index 3d6d0330..5e639133 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1669,13 +1669,13 @@ def test_suggest_facets_off(): assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] -@pytest.mark.parametrize("nofacets", (True, False)) -def test_nofacets(app_client, nofacets): +@pytest.mark.parametrize("nofacet", (True, False)) +def test_nofacet(app_client, nofacet): path = "/fixtures/facetable.json?_facet=state" - if nofacets: - path += "&_nofacets=1" + if nofacet: + path += "&_nofacet=1" response = app_client.get(path) - if nofacets: + if nofacet: assert response.json["suggested_facets"] == [] assert response.json["facet_results"] == {} else: diff --git a/tests/test_csv.py b/tests/test_csv.py index 30afbd9e..40549fd8 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -105,8 +105,8 @@ def test_custom_sql_csv_blob_columns(app_client): assert "text/plain; charset=utf-8" == response.headers["content-type"] assert response.text == ( "rowid,data\r\n" - '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacets=1&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' - '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacets=1&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' + '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacet=1&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' + '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacet=1&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' "3,\r\n" ) From fd368d3b2c5a5d9c3e10a21638f6ea9a71471b52 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 09:12:32 -0700 Subject: [PATCH 0340/1866] New _nocount=1 option, used to speed up CSVs - closes #1353 --- datasette/views/base.py | 15 +++++++++++---- datasette/views/table.py | 6 +++++- docs/json_api.rst | 3 +++ tests/test_api.py | 9 +++++++++ tests/test_csv.py | 6 ++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index b8c581fc..26edfde5 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -263,12 +263,19 @@ class DataView(BaseView): async def as_csv(self, request, database, hash, **kwargs): stream = request.args.get("_stream") - # Do not calculate facets: - if not request.args.get("_nofacet"): + # Do not calculate facets or counts: + extra_parameters = [ + "{}=1".format(key) + for key in ("_nofacet", "_nocount") + if not request.args.get(key) + ] + if extra_parameters: if not request.query_string: - new_query_string = "_nofacet=1" + new_query_string = "&".join(extra_parameters) else: - new_query_string = request.query_string + "&_nofacet=1" + new_query_string = ( + request.query_string + "&" + "&".join(extra_parameters) + ) new_scope = dict( request.scope, query_string=new_query_string.encode("latin-1") ) diff --git a/datasette/views/table.py b/datasette/views/table.py index 7fbf670b..d47865f0 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -697,7 +697,11 @@ class TableView(RowTableShared): except KeyError: pass - if count_sql and filtered_table_rows_count is None: + if ( + count_sql + and filtered_table_rows_count is None + and not request.args.get("_nocount") + ): try: count_rows = list(await db.execute(count_sql, from_sql_params)) filtered_table_rows_count = count_rows[0][0] diff --git a/docs/json_api.rst b/docs/json_api.rst index f1c347b7..660fbc1c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -386,6 +386,9 @@ Special table arguments ``?_nofacet=1`` Disable all facets and facet suggestions for this page, including any defined by :ref:`facets_metadata`. +``?_nocount=1`` + Disable the ``select count(*)`` query used on this page - a count of ``None`` will be returned instead. + ``?_trace=1`` Turns on tracing for this page: SQL queries executed during the request will be gathered and included in the response, either in a new ``"_traces"`` key diff --git a/tests/test_api.py b/tests/test_api.py index 5e639133..49b3bbe9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1683,6 +1683,15 @@ def test_nofacet(app_client, nofacet): assert response.json["facet_results"] != {} +@pytest.mark.parametrize("nocount,expected_count", ((True, None), (False, 15))) +def test_nocount(app_client, nocount, expected_count): + path = "/fixtures/facetable.json" + if nocount: + path += "?_nocount=1" + response = app_client.get(path) + assert response.json["filtered_table_rows_count"] == expected_count + + def test_expand_labels(app_client): response = app_client.get( "/fixtures/facetable.json?_shape=object&_labels=1&_size=2" diff --git a/tests/test_csv.py b/tests/test_csv.py index 40549fd8..02fe5766 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -175,3 +175,9 @@ def test_table_csv_stream_does_not_calculate_facets(app_client): response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") soup = Soup(response.text, "html.parser") assert "select content, count(*) as n" not in soup.find("pre").text + + +def test_table_csv_stream_does_not_calculate_counts(app_client): + response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") + soup = Soup(response.text, "html.parser") + assert "select count(*)" not in soup.find("pre").text From ff45ed0ce5e1f151f24f089c6b78ab7f7a5cd0dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 09:16:58 -0700 Subject: [PATCH 0341/1866] Updated --help output for latest Click, closes #1354 --- docs/datasette-package-help.txt | 4 +--- docs/datasette-publish-cloudrun-help.txt | 2 -- docs/datasette-publish-heroku-help.txt | 3 --- docs/datasette-serve-help.txt | 8 +------- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/docs/datasette-package-help.txt b/docs/datasette-package-help.txt index 5f5ce070..7cfac1b1 100644 --- a/docs/datasette-package-help.txt +++ b/docs/datasette-package-help.txt @@ -7,7 +7,6 @@ Usage: datasette package [OPTIONS] FILES... Options: -t, --tag TEXT Name for the resulting Docker container, can optionally use name:tag format - -m, --metadata FILENAME Path to JSON/YAML file containing metadata to publish --extra-options TEXT Extra options to pass to datasette serve --branch TEXT Install datasette from a GitHub branch e.g. main @@ -19,8 +18,7 @@ Options: --version-note TEXT Additional note to show on /-/versions --secret TEXT Secret used for signing secure values, such as signed cookies - - -p, --port INTEGER RANGE Port to run the server on, defaults to 8001 + -p, --port INTEGER RANGE Port to run the server on, defaults to 8001 [1<=x<=65535] --title TEXT Title for metadata --license TEXT License label for metadata --license_url TEXT License URL for metadata diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index c706d921..3d05efb6 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -13,11 +13,9 @@ Options: --plugin-secret ... Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions --secret TEXT Secret used for signing secure values, such as signed cookies - --title TEXT Title for metadata --license TEXT License label for metadata --license_url TEXT License URL for metadata diff --git a/docs/datasette-publish-heroku-help.txt b/docs/datasette-publish-heroku-help.txt index c4b852de..9d633e95 100644 --- a/docs/datasette-publish-heroku-help.txt +++ b/docs/datasette-publish-heroku-help.txt @@ -13,11 +13,9 @@ Options: --plugin-secret ... Secrets to pass to plugins, e.g. --plugin-secret datasette-auth-github client_id xxx - --version-note TEXT Additional note to show on /-/versions --secret TEXT Secret used for signing secure values, such as signed cookies - --title TEXT Title for metadata --license TEXT License label for metadata --license_url TEXT License URL for metadata @@ -28,5 +26,4 @@ Options: -n, --name TEXT Application name to use when deploying --tar TEXT --tar option to pass to Heroku, e.g. --tar=/usr/local/bin/gtar - --help Show this message and exit. diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 8f770afb..db51dd80 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -10,13 +10,10 @@ Options: connections from the local machine will be allowed. Use 0.0.0.0 to listen to all IPs and allow access from other machines. - -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to automatically - assign an available port. - + assign an available port. [0<=x<=65535] --reload Automatically reload if code or metadata change detected - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: * --load-extension TEXT Path to a SQLite extension to load --inspect-file TEXT Path to JSON file created using "datasette inspect" @@ -27,15 +24,12 @@ Options: --memory Make /_memory database available --config CONFIG Deprecated: set config option using configname:value. Use --setting instead. - --setting SETTING... Setting, see docs.datasette.io/en/stable/config.html --secret TEXT Secret used for signing secure values, such as signed cookies - --root Output URL that sets a cookie authenticating the root user --get TEXT Run an HTTP GET request against this path, print results and exit - --version-note TEXT Additional note to show on /-/versions --help-config Show available config options --pdb Launch debugger on any errors From a18e8641bc33e51b265855bc6e8a1939597b3a76 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 15:35:33 -0700 Subject: [PATCH 0342/1866] Don't reflect nofacet=1 and nocount=1 in BLOB URLs, refs #1353 --- datasette/views/base.py | 5 ++++- tests/test_csv.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 26edfde5..e2583034 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -369,7 +369,7 @@ class DataView(BaseView): ) else: # Otherwise generate URL for this query - cell = self.ds.absolute_url( + url = self.ds.absolute_url( request, path_with_format( request=request, @@ -383,6 +383,9 @@ class DataView(BaseView): replace_format="csv", ), ) + cell = url.replace("&_nocount=1", "").replace( + "&_nofacet=1", "" + ) new_row.append(cell) row = new_row if not expanded_columns: diff --git a/tests/test_csv.py b/tests/test_csv.py index 02fe5766..01f739e2 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -105,8 +105,8 @@ def test_custom_sql_csv_blob_columns(app_client): assert "text/plain; charset=utf-8" == response.headers["content-type"] assert response.text == ( "rowid,data\r\n" - '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacet=1&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' - '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_nofacet=1&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' + '1,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d"\r\n' + '2,"http://localhost/fixtures.blob?sql=select+rowid,+data+from+binary_data&_blob_column=data&_blob_hash=b835b0483cedb86130b9a2c280880bf5fadc5318ddf8c18d0df5204d40df1724"\r\n' "3,\r\n" ) From 0539bf0816b58c7f0ba769331f1509656bff3619 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 19:53:00 -0700 Subject: [PATCH 0343/1866] Don't execute facets/counts for _shape=array or object, closes #263 --- datasette/views/table.py | 17 ++++++++++------- tests/test_api.py | 5 +++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d47865f0..b51d5e5e 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -379,6 +379,13 @@ class TableView(RowTableShared): if is_view: order_by = "" + nocount = request.args.get("_nocount") + nofacet = request.args.get("_nofacet") + + if request.args.get("_shape") in ("array", "object"): + nocount = True + nofacet = True + # Ensure we don't drop anything with an empty value e.g. ?name__exact= args = MultiParams( urllib.parse.parse_qs(request.query_string, keep_blank_values=True) @@ -697,11 +704,7 @@ class TableView(RowTableShared): except KeyError: pass - if ( - count_sql - and filtered_table_rows_count is None - and not request.args.get("_nocount") - ): + if count_sql and filtered_table_rows_count is None and not nocount: try: count_rows = list(await db.execute(count_sql, from_sql_params)) filtered_table_rows_count = count_rows[0][0] @@ -735,7 +738,7 @@ class TableView(RowTableShared): ) ) - if not request.args.get("_nofacet"): + if not nofacet: for facet in facet_instances: ( instance_facet_results, @@ -833,7 +836,7 @@ class TableView(RowTableShared): self.ds.setting("suggest_facets") and self.ds.setting("allow_facet") and not _next - and not request.args.get("_nofacet") + and not nofacet ): for facet in facet_instances: suggested_facets.extend(await facet.suggest()) diff --git a/tests/test_api.py b/tests/test_api.py index 49b3bbe9..078aad35 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1692,6 +1692,11 @@ def test_nocount(app_client, nocount, expected_count): assert response.json["filtered_table_rows_count"] == expected_count +def test_nocount_nofacet_if_shape_is_object(app_client): + response = app_client.get("/fixtures/facetable.json?_trace=1&_shape=object") + assert "count(*)" not in response.text + + def test_expand_labels(app_client): response = app_client.get( "/fixtures/facetable.json?_shape=object&_labels=1&_size=2" From 03b35d70e281ea48bd9b8058738ed87b13cea2de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jun 2021 19:56:44 -0700 Subject: [PATCH 0344/1866] Bump black from 21.5b1 to 21.5b2 (#1352) Bumps [black](https://github.com/psf/black) from 21.5b1 to 21.5b2. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 60a94a5e..e66fefc3 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup( "pytest-xdist>=2.2.1,<2.3", "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", - "black==21.5b1", + "black==21.5b2", "pytest-timeout>=1.4.2,<1.5", "trustme>=0.7,<0.8", ], From 807de378d08752a0f05bb1b980a0a62620a70520 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 20:03:07 -0700 Subject: [PATCH 0345/1866] /-/databases and homepage maintain connection order, closes #1216 --- datasette/app.py | 2 +- tests/fixtures.py | 3 ++- tests/test_api.py | 2 +- tests/test_html.py | 6 +++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 957ced7c..018a8d5b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -646,7 +646,7 @@ class Datasette: "is_memory": d.is_memory, "hash": d.hash, } - for name, d in sorted(self.databases.items(), key=lambda p: p[1].name) + for name, d in self.databases.items() if name != "_internal" ] diff --git a/tests/fixtures.py b/tests/fixtures.py index 5730c1bf..2690052a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -126,7 +126,8 @@ def make_app_client( for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) sqlite3.connect(extra_filepath).executescript(extra_sql) - files.append(extra_filepath) + # Insert at start to help test /-/databases ordering: + files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) config = config or {} for key, value in { diff --git a/tests/test_api.py b/tests/test_api.py index 078aad35..3b789bb7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1918,7 +1918,7 @@ def test_database_with_space_in_name(app_client_two_attached_databases, path): def test_common_prefix_database_names(app_client_conflicting_database_names): # https://github.com/simonw/datasette/issues/597 - assert ["fixtures", "foo", "foo-bar"] == [ + assert ["foo-bar", "foo", "fixtures"] == [ d["name"] for d in app_client_conflicting_database_names.get("/-/databases.json").json ] diff --git a/tests/test_html.py b/tests/test_html.py index 4f2cc8ad..fd60cdc9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -29,11 +29,11 @@ def test_homepage(app_client_two_attached_databases): ) # Should be two attached databases assert [ - {"href": "/fixtures", "text": "fixtures"}, {"href": r"/extra%20database", "text": "extra database"}, + {"href": "/fixtures", "text": "fixtures"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] - # The first attached database should show count text and attached tables - h2 = soup.select("h2")[1] + # Database should show count text and attached tables + h2 = soup.select("h2")[0] assert "extra database" == h2.text.strip() counts_p, links_p = h2.find_all_next("p")[:2] assert ( From 0f1e47287cf2185e140bd87a03c985c2a7afb450 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 20:27:04 -0700 Subject: [PATCH 0346/1866] Fixed bug with detect_fts for table with single quote in name, closes #1257 --- datasette/utils/__init__.py | 2 +- tests/test_utils.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index dd47771f..73122976 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -547,7 +547,7 @@ def detect_fts_sql(table): ) ) """.format( - table=table + table=table.replace("'", "''") ) diff --git a/tests/test_utils.py b/tests/test_utils.py index ecef6f7a..be3daf2e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -200,6 +200,22 @@ def test_detect_fts(open_quote, close_quote): assert "Street_Tree_List_fts" == utils.detect_fts(conn, "Street_Tree_List") +@pytest.mark.parametrize("table", ("regular", "has'single quote")) +def test_detect_fts_different_table_names(table): + sql = """ + CREATE TABLE [{table}] ( + "TreeID" INTEGER, + "qSpecies" TEXT + ); + CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}"); + """.format( + table=table + ) + conn = utils.sqlite3.connect(":memory:") + conn.executescript(sql) + assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) + + @pytest.mark.parametrize( "url,expected", [ From 9552414e1f968c6fc704031cec349c05e6bc2371 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 23:46:20 -0400 Subject: [PATCH 0347/1866] Re-display user's query with an error message if an error occurs (#1346) * Ignore _shape when returning errors --- datasette/renderer.py | 4 ++++ datasette/templates/query.html | 5 ++++- datasette/views/base.py | 21 +++++++++++++++++---- datasette/views/database.py | 25 ++++++++++++++++++------- tests/test_canned_queries.py | 2 +- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 66ac169b..45089498 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -29,6 +29,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): def json_renderer(args, data, view_name): """Render a response as JSON""" status_code = 200 + # Handle the _json= parameter which may modify data["rows"] json_cols = [] if "_json" in args: @@ -44,6 +45,9 @@ def json_renderer(args, data, view_name): # Deal with the _shape option shape = args.get("_shape", "arrays") + # if there's an error, ignore the shape entirely + if data.get("error"): + shape = "arrays" next_url = data.get("next_url") diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 9b3fff25..633e53b4 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -33,7 +33,10 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %} {% if hide_sql %}(show){% else %}(hide){% endif %}

    +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} {% if hide_sql %}(show){% else %}(hide){% endif %}{% endif %}

    + {% if query_error %} +

    {{ query_error }}

    + {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %}

    diff --git a/datasette/views/base.py b/datasette/views/base.py index e2583034..94f54787 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -294,6 +294,8 @@ class DataView(BaseView): ) if isinstance(response_or_template_contexts, Response): return response_or_template_contexts + elif len(response_or_template_contexts) == 4: + data, _, _, _ = response_or_template_contexts else: data, _, _ = response_or_template_contexts except (sqlite3.OperationalError, InvalidSql) as e: @@ -467,7 +469,7 @@ class DataView(BaseView): extra_template_data = {} start = time.perf_counter() - status_code = 200 + status_code = None templates = [] try: response_or_template_contexts = await self.data( @@ -475,7 +477,14 @@ class DataView(BaseView): ) if isinstance(response_or_template_contexts, Response): return response_or_template_contexts - + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts else: data, extra_template_data, templates = response_or_template_contexts except QueryInterrupted: @@ -542,12 +551,15 @@ class DataView(BaseView): if isinstance(result, dict): r = Response( body=result.get("body"), - status=result.get("status_code", 200), + status=result.get("status_code", status_code or 200), content_type=result.get("content_type", "text/plain"), headers=result.get("headers"), ) elif isinstance(result, Response): r = result + if status_code is not None: + # Over-ride the status code + r.status = status_code else: assert False, f"{result} should be dict or Response" else: @@ -607,7 +619,8 @@ class DataView(BaseView): if "metadata" not in context: context["metadata"] = self.ds.metadata r = await self.render(templates, request=request, context=context) - r.status = status_code + if status_code is not None: + r.status = status_code ttl = request.args.get("_ttl", None) if ttl is None or not ttl.isdigit(): diff --git a/datasette/views/database.py b/datasette/views/database.py index 96b2ca91..58168ed7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -14,6 +14,7 @@ from datasette.utils import ( path_with_added_args, path_with_format, path_with_removed_args, + sqlite3, InvalidSql, ) from datasette.utils.asgi import AsgiFileDownload, Response, Forbidden @@ -239,6 +240,8 @@ class QueryView(DataView): templates = [f"query-{to_css_class(database)}.html", "query.html"] + query_error = None + # Execute query - as write or as read if write: if request.method == "POST": @@ -320,10 +323,15 @@ class QueryView(DataView): params_for_query = MagicParameters(params, request, self.ds) else: params_for_query = params - results = await self.ds.execute( - database, sql, params_for_query, truncate=True, **extra_args - ) - columns = [r[0] for r in results.description] + try: + results = await self.ds.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] if canned_query: templates.insert( @@ -337,7 +345,7 @@ class QueryView(DataView): async def extra_template(): display_rows = [] - for row in results.rows: + for row in results.rows if results else []: display_row = [] for column, value in zip(results.columns, row): display_value = value @@ -423,17 +431,20 @@ class QueryView(DataView): return ( { + "ok": not query_error, "database": database, "query_name": canned_query, - "rows": results.rows, - "truncated": results.truncated, + "rows": results.rows if results else [], + "truncated": results.truncated if results else False, "columns": columns, "query": {"sql": sql, "params": params}, + "error": str(query_error) if query_error else None, "private": private, "allow_execute_sql": allow_execute_sql, }, extra_template, templates, + 400 if query_error else 200, ) diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 65f23cc7..4186a97c 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -352,5 +352,5 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c response = magic_parameters_client.get( "/data.json?sql=select+:_header_host&_shape=array" ) - assert 500 == response.status + assert 400 == response.status assert "You did not supply a value for binding 1." == response.json["error"] From ea5b2378007ef524f7a17989c8df54a76a001e49 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 20:59:29 -0700 Subject: [PATCH 0348/1866] Show error message on bad query, closes #619 --- datasette/templates/query.html | 4 ++-- tests/test_html.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 633e53b4..8b6ad138 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -34,8 +34,8 @@

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} {% if hide_sql %}(show){% else %}(hide){% endif %}{% endif %}

    - {% if query_error %} -

    {{ query_error }}

    + {% if error %} +

    {{ error }}

    {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} diff --git a/tests/test_html.py b/tests/test_html.py index fd60cdc9..5fca76c3 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1402,6 +1402,16 @@ def test_zero_results(app_client, path): assert 1 == len(soup.select("p.zero-results")) +def test_query_error(app_client): + response = app_client.get("/fixtures?sql=select+*+from+notatable") + html = response.text + assert '

    no such table: notatable

    ' in html + assert ( + '' + in html + ) + + def test_config_template_debug_on(): with make_app_client(config={"template_debug": True}) as client: response = client.get("/fixtures/facetable?_context=1") From f40d1b99d67b0da4f3aff5b3483f4e09db7e8e6b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 21:09:10 -0700 Subject: [PATCH 0349/1866] Don't show '0 results' on error page, refs #619 --- datasette/templates/query.html | 2 +- tests/test_html.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8b6ad138..b6c74883 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -80,7 +80,7 @@
    {% if td == None %}{{ " "|safe }}{% else %}{{ td }}{% endif %}{{ td }}
    11<Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes>22<Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes>3<Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes>\xa01hello\xa01-\xa0312\xa0\xa0\xa01131{}{}{}a{}b{}c{}{i}{i}a{i}b{i}c{i}a,bhello\xa01-\xa031ab2\xa0\xa0\xa0\xa0\xa0131ab
    {% else %} - {% if not canned_write %} + {% if not canned_write and not error %}

    0 results

    {% endif %} {% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 5fca76c3..90373c28 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1410,6 +1410,7 @@ def test_query_error(app_client): '' in html ) + assert "0 results" not in html def test_config_template_debug_on(): From 0f41db1ba8a8a49a4adc1046a25ccf32790e863f Mon Sep 17 00:00:00 2001 From: Guy Freeman Date: Wed, 2 Jun 2021 07:25:27 +0300 Subject: [PATCH 0350/1866] Avoid error sorting by relationships if related tables are not allowed Refs #1306 --- datasette/views/index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datasette/views/index.py b/datasette/views/index.py index b6b8cbe5..8ac117a6 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -78,8 +78,9 @@ class IndexView(BaseView): # We will be sorting by number of relationships, so populate that field all_foreign_keys = await db.get_all_foreign_keys() for table, foreign_keys in all_foreign_keys.items(): - count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) - tables[table]["num_relationships_for_sorting"] = count + if table in tables.keys(): + count = len(foreign_keys["incoming"] + foreign_keys["outgoing"]) + tables[table]["num_relationships_for_sorting"] = count hidden_tables = [t for t in tables.values() if t["hidden"]] visible_tables = [t for t in tables.values() if not t["hidden"]] From 80d8b0eb415faf5caadd7cc7036407e6ee55bd44 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 21:26:25 -0700 Subject: [PATCH 0351/1866] Test demonstrating fixed #1305, refs #1306 --- tests/test_html.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_html.py b/tests/test_html.py index 90373c28..8bc53339 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1685,3 +1685,22 @@ def test_facet_more_links( assert facet_truncated.find("a")["href"] == expected_ellipses_url else: assert facet_truncated.find("a") is None + + +def test_unavailable_table_does_not_break_sort_relationships(): + # https://github.com/simonw/datasette/issues/1305 + with make_app_client( + metadata={ + "databases": { + "fixtures": { + "tables": { + "foreign_key_references": { + "allow": False + } + } + } + } + } + ) as client: + response = client.get("/?_sort=relationships") + assert response.status == 200 From d5d387abfe68ea546c53698ebb2b8eeeb4d32c3f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Jun 2021 21:30:44 -0700 Subject: [PATCH 0352/1866] Applied Black, refs #1305 --- tests/test_html.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_html.py b/tests/test_html.py index 8bc53339..31bb6667 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1692,13 +1692,7 @@ def test_unavailable_table_does_not_break_sort_relationships(): with make_app_client( metadata={ "databases": { - "fixtures": { - "tables": { - "foreign_key_references": { - "allow": False - } - } - } + "fixtures": {"tables": {"foreign_key_references": {"allow": False}}} } } ) as client: From f78ebdc04537a6102316d6dbbf6c887565806078 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 10:00:30 -0700 Subject: [PATCH 0353/1866] Better "uploading and publishing your own CSV data" link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f3c9a94..5682f59e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Datasette is a tool for exploring and publishing data. It helps people take data Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world. -[Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://simonwillison.net/2019/Apr/23/datasette-glitch/). +[Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch). * [datasette.io](https://datasette.io/) is the official project website * Latest [Datasette News](https://datasette.io/news) From 6e9b07be92905011211d8df7a872fb7c1f2737b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 2 Jun 2021 21:45:03 -0700 Subject: [PATCH 0354/1866] More inclusive language --- datasette/cli.py | 2 +- datasette/facets.py | 2 +- datasette/views/base.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 71bbc353..12ee92c3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -540,7 +540,7 @@ def serve( # Run the "startup" plugin hooks asyncio.get_event_loop().run_until_complete(ds.invoke_startup()) - # Run async sanity checks - but only if we're not under pytest + # Run async soundness checks - but only if we're not under pytest asyncio.get_event_loop().run_until_complete(check_databases(ds)) if get: diff --git a/datasette/facets.py b/datasette/facets.py index 9d95d0f3..250734fd 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -304,7 +304,7 @@ class ArrayFacet(Facet): ) types = tuple(r[0] for r in results.rows) if types in (("array",), ("array", None)): - # Now sanity check that first 100 arrays contain only strings + # Now check that first 100 arrays contain only strings first_100 = [ v[0] for v in await self.ds.execute( diff --git a/datasette/views/base.py b/datasette/views/base.py index 94f54787..1a03b97f 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -281,7 +281,7 @@ class DataView(BaseView): ) request.scope = new_scope if stream: - # Some quick sanity checks + # Some quick soundness checks if not self.ds.setting("allow_csv_stream"): raise BadRequest("CSV streaming is disabled") if request.args.get("_next"): From a63412152518581c6a3d4e142b937e27dabdbfdb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 11:59:54 -0700 Subject: [PATCH 0355/1866] Make custom pages compatible with base_url setting Closes #1238 - base_url no longer causes custom page routing to fail - new route_path key in request.scope storing the path that was used for routing with the base_url prefix stripped - TestClient used by tests now avoids accidentally double processing of the base_url prefix --- datasette/app.py | 17 ++++++++++------- datasette/utils/testing.py | 1 + tests/test_custom_pages.py | 16 +++++++++++++++- tests/test_html.py | 1 + 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 018a8d5b..c0e8ad01 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1089,6 +1089,7 @@ class DatasetteRouter: base_url = self.ds.setting("base_url") if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] + scope = dict(scope, route_path=path) request = Request(scope, receive) # Populate request_messages if ds_messages cookie is present try: @@ -1143,9 +1144,8 @@ class DatasetteRouter: await asgi_send_redirect(send, path.decode("latin1")) else: # Is there a pages/* template matching this path? - template_path = ( - os.path.join("pages", *request.scope["path"].split("/")) + ".html" - ) + route_path = request.scope.get("route_path", request.scope["path"]) + template_path = os.path.join("pages", *route_path.split("/")) + ".html" try: template = self.ds.jinja_env.select_template([template_path]) except TemplateNotFound: @@ -1153,7 +1153,7 @@ class DatasetteRouter: if template is None: # Try for a pages/blah/{name}.html template match for regex, wildcard_template in self.page_routes: - match = regex.match(request.scope["path"]) + match = regex.match(route_path) if match is not None: context.update(match.groupdict()) template = wildcard_template @@ -1356,8 +1356,8 @@ class DatasetteClient: self.ds = ds self.app = ds.app() - def _fix(self, path): - if not isinstance(path, PrefixedUrlString): + def _fix(self, path, avoid_path_rewrites=False): + if not isinstance(path, PrefixedUrlString) and not avoid_path_rewrites: path = self.ds.urls.path(path) if path.startswith("/"): path = f"http://localhost{path}" @@ -1392,5 +1392,8 @@ class DatasetteClient: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): + avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient(app=self.app) as client: - return await client.request(method, self._fix(path), **kwargs) + return await client.request( + method, self._fix(path, avoid_path_rewrites), **kwargs + ) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 57b19ea5..a169a83d 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -140,6 +140,7 @@ class TestClient: method, path, allow_redirects=allow_redirects, + avoid_path_rewrites=True, cookies=cookies, headers=headers, content=post_body, diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 6a231920..5a71f56d 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -2,11 +2,19 @@ import pathlib import pytest from .fixtures import make_app_client +TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates") + @pytest.fixture(scope="session") def custom_pages_client(): + with make_app_client(template_dir=TEST_TEMPLATE_DIRS) as client: + yield client + + +@pytest.fixture(scope="session") +def custom_pages_client_with_base_url(): with make_app_client( - template_dir=str(pathlib.Path(__file__).parent / "test_templates") + template_dir=TEST_TEMPLATE_DIRS, config={"base_url": "/prefix/"} ) as client: yield client @@ -23,6 +31,12 @@ def test_request_is_available(custom_pages_client): assert "path:/request" == response.text +def test_custom_pages_with_base_url(custom_pages_client_with_base_url): + response = custom_pages_client_with_base_url.get("/prefix/request") + assert 200 == response.status + assert "path:/prefix/request" == response.text + + def test_custom_pages_nested(custom_pages_client): response = custom_pages_client.get("/nested/nest") assert 200 == response.status diff --git a/tests/test_html.py b/tests/test_html.py index 31bb6667..f1d4bd70 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1523,6 +1523,7 @@ def test_base_url_config(app_client_base_url_prefix, path): and href not in { "https://datasette.io/", + "https://github.com/simonw/datasette", "https://github.com/simonw/datasette/blob/main/LICENSE", "https://github.com/simonw/datasette/blob/main/tests/fixtures.py", "/login-as-root", # Only used for the latest.datasette.io demo From 368aa5f1b16ca35f82d90ff747023b9a2bfa27c1 Mon Sep 17 00:00:00 2001 From: louispotok Date: Sun, 6 Jun 2021 02:48:51 +0700 Subject: [PATCH 0356/1866] Update docs: explain allow_download setting (#1291) * Update docs: explain allow_download setting This fixes one possible source of confusion seen in #502 and clarifies when database downloads will be shown and allowed. --- docs/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings.rst b/docs/settings.rst index af8e4406..db17a45e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -163,7 +163,7 @@ Should Datasette calculate suggested facets? On by default, turn this off like s allow_download ~~~~~~~~~~~~~~ -Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default - to disable database downloads, use the following:: +Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default. However, databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following:: datasette mydatabase.db --setting allow_download off From ff29dd55fafd7c3d27bd30f40945847aa4278309 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 13:15:58 -0700 Subject: [PATCH 0357/1866] ?_trace=1 now depends on trace_debug setting, closes #1359 --- .github/workflows/deploy-latest.yml | 2 +- datasette/app.py | 20 +++++++++++++------- docs/json_api.rst | 18 ++++++++++-------- docs/settings.rst | 16 ++++++++++++++++ tests/fixtures.py | 6 ++++++ tests/test_api.py | 20 ++++++++++++++++---- tests/test_csv.py | 13 +++++++------ 7 files changed, 69 insertions(+), 26 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 43e46fb4..d9f23f7d 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -53,7 +53,7 @@ jobs: --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --crossdb" \ + --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ --install=pysqlite3-binary \ --service=datasette-latest # Deploy docs.db to a different service diff --git a/datasette/app.py b/datasette/app.py index c0e8ad01..d85517e6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -162,6 +162,11 @@ SETTINGS = ( False, "Allow display of template debug information with ?_context=1", ), + Setting( + "trace_debug", + False, + "Allow display of SQL trace debug information with ?_trace=1", + ), Setting("base_url", "/", "Datasette URLs should use this base path"), ) @@ -1041,14 +1046,15 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) + asgi = asgi_csrf.asgi_csrf( + DatasetteRouter(self, routes), + signing_secret=self._secret, + cookie_name="ds_csrftoken", + ) + if self.setting("trace_debug"): + asgi = AsgiTracer(asgi) asgi = AsgiLifespan( - AsgiTracer( - asgi_csrf.asgi_csrf( - DatasetteRouter(self, routes), - signing_secret=self._secret, - cookie_name="ds_csrftoken", - ) - ), + asgi, on_startup=setup_db, ) for wrapper in pm.hook.asgi_wrapper(datasette=self): diff --git a/docs/json_api.rst b/docs/json_api.rst index 660fbc1c..09cac1f9 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -206,6 +206,16 @@ query string arguments: For how many seconds should this response be cached by HTTP proxies? Use ``?_ttl=0`` to disable HTTP caching entirely for this request. +``?_trace=1`` + Turns on tracing for this page: SQL queries executed during the request will + be gathered and included in the response, either in a new ``"_traces"`` key + for JSON responses or at the bottom of the page if the response is in HTML. + + The structure of the data returned here should be considered highly unstable + and very likely to change. + + Only available if the :ref:`setting_trace_debug` setting is enabled. + .. _table_arguments: Table arguments @@ -389,14 +399,6 @@ Special table arguments ``?_nocount=1`` Disable the ``select count(*)`` query used on this page - a count of ``None`` will be returned instead. -``?_trace=1`` - Turns on tracing for this page: SQL queries executed during the request will - be gathered and included in the response, either in a new ``"_traces"`` key - for JSON responses or at the bottom of the page if the response is in HTML. - - The structure of the data returned here should be considered highly unstable - and very likely to change. - .. _expand_foreign_keys: Expanding foreign key references diff --git a/docs/settings.rst b/docs/settings.rst index db17a45e..c246d33a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -286,6 +286,22 @@ Some examples: * https://latest.datasette.io/fixtures?_context=1 * https://latest.datasette.io/fixtures/roadside_attractions?_context=1 +.. _setting_trace_debug: + +trace_debug +~~~~~~~~~~~ + +This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page. + +Enable it like this:: + + datasette mydatabase.db --setting trace_debug 1 + +Some examples: + +* https://latest.datasette.io/?_trace=1 +* https://latest.datasette.io/fixtures/roadside_attractions?_trace=1 + .. _setting_base_url: base_url diff --git a/tests/fixtures.py b/tests/fixtures.py index 2690052a..cdd2e987 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -214,6 +214,12 @@ def app_client_with_hash(): yield client +@pytest.fixture(scope="session") +def app_client_with_trace(): + with make_app_client(config={"trace_debug": True}, is_immutable=True) as client: + yield client + + @pytest.fixture(scope="session") def app_client_shorter_time_limit(): with make_app_client(20) as client: diff --git a/tests/test_api.py b/tests/test_api.py index 3b789bb7..e5e609d6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,6 +15,7 @@ from .fixtures import ( # noqa app_client_conflicting_database_names, app_client_with_cors, app_client_with_dot, + app_client_with_trace, app_client_immutable_and_inspect_file, generate_compound_rows, generate_sortable_rows, @@ -1422,6 +1423,7 @@ def test_settings_json(app_client): "force_https_urls": False, "hash_urls": False, "template_debug": False, + "trace_debug": False, "base_url": "/", } == response.json @@ -1692,8 +1694,10 @@ def test_nocount(app_client, nocount, expected_count): assert response.json["filtered_table_rows_count"] == expected_count -def test_nocount_nofacet_if_shape_is_object(app_client): - response = app_client.get("/fixtures/facetable.json?_trace=1&_shape=object") +def test_nocount_nofacet_if_shape_is_object(app_client_with_trace): + response = app_client_with_trace.get( + "/fixtures/facetable.json?_trace=1&_shape=object" + ) assert "count(*)" not in response.text @@ -1863,9 +1867,17 @@ def test_custom_query_with_unicode_characters(app_client): assert [{"id": 1, "name": "San Francisco"}] == response.json -def test_trace(app_client): - response = app_client.get("/fixtures/simple_primary_key.json?_trace=1") +@pytest.mark.parametrize("trace_debug", (True, False)) +def test_trace(trace_debug): + with make_app_client(config={"trace_debug": trace_debug}) as client: + response = client.get("/fixtures/simple_primary_key.json?_trace=1") + assert response.status == 200 + data = response.json + if not trace_debug: + assert "_trace" not in data + return + assert "_trace" in data trace_info = data["_trace"] assert isinstance(trace_info["request_duration_ms"], float) diff --git a/tests/test_csv.py b/tests/test_csv.py index 01f739e2..3debf320 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -3,6 +3,7 @@ from .fixtures import ( # noqa app_client, app_client_csv_max_mb_one, app_client_with_cors, + app_client_with_trace, ) EXPECTED_TABLE_CSV = """id,content @@ -160,8 +161,8 @@ def test_table_csv_stream(app_client): assert 1002 == len([b for b in response.body.split(b"\r\n") if b]) -def test_csv_trace(app_client): - response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") +def test_csv_trace(app_client_with_trace): + response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1") assert response.headers["content-type"] == "text/html; charset=utf-8" soup = Soup(response.text, "html.parser") assert ( @@ -171,13 +172,13 @@ def test_csv_trace(app_client): assert "select id, content from simple_primary_key" in soup.find("pre").text -def test_table_csv_stream_does_not_calculate_facets(app_client): - response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") +def test_table_csv_stream_does_not_calculate_facets(app_client_with_trace): + response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1") soup = Soup(response.text, "html.parser") assert "select content, count(*) as n" not in soup.find("pre").text -def test_table_csv_stream_does_not_calculate_counts(app_client): - response = app_client.get("/fixtures/simple_primary_key.csv?_trace=1") +def test_table_csv_stream_does_not_calculate_counts(app_client_with_trace): + response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1") soup = Soup(response.text, "html.parser") assert "select count(*)" not in soup.find("pre").text From 8f311d6c1d9f73f4ec643009767749c17b5ca5dd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 14:49:16 -0700 Subject: [PATCH 0358/1866] Correctly escape output of ?_trace, refs #1360 --- datasette/tracer.py | 3 ++- tests/test_html.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 772f0405..62c3c90c 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -1,5 +1,6 @@ import asyncio from contextlib import contextmanager +from markupsafe import escape import time import json import traceback @@ -123,7 +124,7 @@ class AsgiTracer: except IndexError: content_type = "" if "text/html" in content_type and b"" in accumulated_body: - extra = json.dumps(trace_info, indent=2) + extra = escape(json.dumps(trace_info, indent=2)) extra_html = f"
    {extra}
    ".encode("utf8") accumulated_body = accumulated_body.replace(b"", extra_html) elif "json" in content_type and accumulated_body.startswith(b"{"): diff --git a/tests/test_html.py b/tests/test_html.py index f1d4bd70..8714d254 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1699,3 +1699,9 @@ def test_unavailable_table_does_not_break_sort_relationships(): ) as client: response = client.get("/?_sort=relationships") assert response.status == 200 + + +def test_trace_correctly_escaped(app_client): + response = app_client.get("/fixtures?sql=select+'

    Hello'&_trace=1") + assert "select '

    Hello" not in response.text + assert "select '<h1>Hello" in response.text From 58746d3c514004f504223a724e948469a0d4abb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 15:06:52 -0700 Subject: [PATCH 0359/1866] Release 0.57 Refs #263, #615, #619, #1238, #1257, #1305, #1308, #1320, #1332, #1337, #1349, #1353, #1359, #1360 --- datasette/version.py | 2 +- docs/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index cc98e271..93af8b3b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.57a1" +__version__ = "0.57" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e00791f8..842ca839 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,16 +4,45 @@ Changelog ========= -.. _v0_57_a0: +.. _v0_57: -0.57a0 (2021-05-22) +0.57 (2021-06-05) +----------------- + +.. warning:: + This release fixes a `reflected cross-site scripting `__ security hole with the ``?_trace=1`` feature. You should upgrade to this version, or to Datasette 0.56.1, as soon as possible. (:issue:`1360`) + +In addition to the security fix, this release includes ``?_col=`` and ``?_nocol=`` options for controlling which columns are displayed for a table, ``?_facet_size=`` for increasing the number of facet results returned, re-display of your SQL query should an error occur and numerous bug fixes. + +New features +~~~~~~~~~~~~ + +- If an error occurs while executing a user-provided SQL query, that query is now re-displayed in an editable form along with the error message. (:issue:`619`) +- New ``?_col=`` and ``?_nocol=`` parameters to show and hide columns in a table, plus an interface for hiding and showing columns in the column cog menu. (:issue:`615`) +- A new ``?_facet_size=`` parameter for customizing the number of facet results returned on a table or view page. (:issue:`1332`) +- ``?_facet_size=max`` sets that to the maximum, which defaults to 1,000 and is controlled by the the :ref:`setting_max_returned_rows` setting. If facet results are truncated the … at the bottom of the facet list now links to this parameter. (:issue:`1337`) +- ``?_nofacet=1`` option to disable all facet calculations on a page, used as a performance optimization for CSV exports and ``?_shape=array/object``. (:issue:`1349`, :issue:`263`) +- ``?_nocount=1`` option to disable full query result counts. (:issue:`1353`) +- ``?_trace=1`` debugging option is now controlled by the new :ref:`setting_trace_debug` setting, which is turned off by default. (:issue:`1359`) + +Bug fixes and other improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- :ref:`custom_pages` now work correctly when combined with the :ref:`setting_base_url` setting. (:issue:`1238`) +- Fixed intermittent error displaying the index page when the user did not have permission to access one of the tables. Thanks, Guy Freeman. (:issue:`1305`) +- Columns with the name "Link" are no longer incorrectly displayed in bold. (:issue:`1308`) +- Fixed error caused by tables with a single quote in their names. (:issue:`1257`) +- Updated dependencies: ``pytest-asyncio``, ``Black``, ``jinja2``, ``aiofiles``, ``click``, and ``itsdangerous``. +- The official Datasette Docker image now supports ``apt-get install``. (:issue:`1320`) +- The Heroku runtime used by ``datasette publish heroku`` is now ``python-3.8.10``. + +.. _v0_56_1: + +0.56.1 (2021-06-05) ------------------- -Mainly dependency bumps, plus a new ``?_facet_size=`` argument. - -- Updated dependencies: pytest-asyncio, Black, jinja2, aiofiles, itsdangerous -- Fixed bug where columns called "Link" were incorrectly displayed in bold. (:issue:`1308`) -- New ``?_facet_size=`` argument for customizing the number of facet results returned on a page. (:issue:`1332`) +.. warning:: + This release fixes a `reflected cross-site scripting `__ security hole with the ``?_trace=1`` feature. You should upgrade to this version, or to Datasette 0.57, as soon as possible. (:issue:`1360`) .. _v0_56: From 0dfb9241718139f8ad626d22aac25bcebd3a9c9c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 15:55:07 -0700 Subject: [PATCH 0360/1866] Temporarily reverting buildx support I need to push a container for 0.57 using this action, and I'm not ready to ship other architecture builds until we have tested them in #1344. --- .github/workflows/push_docker_tag.yml | 34 +++++++-------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index e61150a5..9a3969f0 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -11,31 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Available platforms - run: echo ${{ steps.buildx.outputs.platforms }} - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - name: Build and push to Docker Hub - run: | - docker buildx build \ - --file Dockerfile . \ - --tag $REPO:${VERSION_TAG} \ - --build-arg VERSION=${VERSION_TAG} \ - --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ - --push env: - REPO: datasetteproject/datasette + DOCKER_USER: ${{ secrets.DOCKER_USER }} + DOCKER_PASS: ${{ secrets.DOCKER_PASS }} VERSION_TAG: ${{ github.event.inputs.version_tag }} + run: |- + docker login -u $DOCKER_USER -p $DOCKER_PASS + export REPO=datasetteproject/datasette + docker build -f Dockerfile \ + -t $REPO:${VERSION_TAG} \ + --build-arg VERSION=${VERSION_TAG} . + docker push $REPO:${VERSION_TAG} From 030deb4b25cda842ff7129ab7c18550c44dd8379 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 5 Jun 2021 16:01:34 -0700 Subject: [PATCH 0361/1866] Try to handle intermittent FileNotFoundError in tests Refs #1361 --- tests/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ad3eb9f1..c6a3eee6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,13 +61,18 @@ def move_to_front(items, test_name): @pytest.fixture def restore_working_directory(tmpdir, request): - previous_cwd = os.getcwd() + try: + previous_cwd = os.getcwd() + except OSError: + # https://github.com/simonw/datasette/issues/1361 + previous_cwd = None tmpdir.chdir() def return_to_previous(): os.chdir(previous_cwd) - request.addfinalizer(return_to_previous) + if previous_cwd is not None: + request.addfinalizer(return_to_previous) @pytest.fixture(scope="session", autouse=True) From 03ec71193b9545536898a4bc7493274fec48bdd7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 6 Jun 2021 15:07:45 -0700 Subject: [PATCH 0362/1866] Don't truncate list of columns on /db page, closes #1364 --- datasette/templates/database.html | 2 +- tests/test_html.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 3fe7c891..2d182d1b 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -71,7 +71,7 @@ {% if show_hidden or not table.hidden %}

    {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}

    -

    {% for column in table.columns[:9] %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}{% if table.columns|length > 9 %}...{% endif %}

    +

    {% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}

    {% if table.count is none %}Many rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}

    {% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 8714d254..ccee8b7e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -101,6 +101,11 @@ def test_database_page_redirects_with_url_hash(app_client_with_hash): def test_database_page(app_client): response = app_client.get("/fixtures") + assert ( + b"

    pk, foreign_key_with_label, foreign_key_with_blank_label, " + b"foreign_key_with_no_label, foreign_key_compound_pk1, " + b"foreign_key_compound_pk2

    " + ) in response.body soup = Soup(response.body, "html.parser") queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") assert queries_ul is not None From f4c5777c7e4ed406313583de09a3bf746552167f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Jun 2021 11:24:14 -0700 Subject: [PATCH 0363/1866] Fix visual glitch in nav menu, closes #1367 --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 7f04a162..ad517c98 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -452,7 +452,8 @@ table a:link { margin-left: -10%; font-size: 0.8em; } -.rows-and-columns td ol,ul { +.rows-and-columns td ol, +.rows-and-columns td ul { list-style: initial; list-style-position: inside; } From a3faf378834cc9793adeb22dee19ef57c417457e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Jun 2021 09:26:45 -0700 Subject: [PATCH 0364/1866] Release 0.57.1 Refs #1364, #1367 --- datasette/version.py | 2 +- docs/changelog.rst | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 93af8b3b..14a7be17 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.57" +__version__ = "0.57.1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 842ca839..89b8fcf5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _v0_57_1: + +0.57.1 (2021-06-08) +------------------- + +- Fixed visual display glitch with global navigation menu. (:issue:`1367`) +- No longer truncates the list of table columns displayed on the ``/database`` page. (:issue:`1364`) + .. _v0_57: 0.57 (2021-06-05) From d23a2671386187f61872b9f6b58e0f80ac61f8fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Jun 2021 21:45:24 -0700 Subject: [PATCH 0365/1866] Make request available to menu plugin hooks, closes #1371 --- datasette/app.py | 4 +++- datasette/hookspecs.py | 6 +++--- datasette/views/database.py | 1 + datasette/views/table.py | 1 + docs/plugin_hooks.rst | 22 +++++++++++++++------- tests/plugins/my_plugin.py | 14 ++++++++++---- tests/plugins/my_plugin_2.py | 7 +++++-- tests/test_plugins.py | 12 ++++++------ 8 files changed, 44 insertions(+), 23 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d85517e6..fc5b7d9d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -833,7 +833,9 @@ class Datasette: async def menu_links(): links = [] for hook in pm.hook.menu_links( - datasette=self, actor=request.actor if request else None + datasette=self, + actor=request.actor if request else None, + request=request or None, ): extra_links = await await_me_maybe(hook) if extra_links: diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 13a10680..579787a2 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -100,15 +100,15 @@ def forbidden(datasette, request, message): @hookspec -def menu_links(datasette, actor): +def menu_links(datasette, actor, request): """Links for the navigation menu""" @hookspec -def table_actions(datasette, actor, database, table): +def table_actions(datasette, actor, database, table, request): """Links for the table actions menu""" @hookspec -def database_actions(datasette, actor, database): +def database_actions(datasette, actor, database, request): """Links for the database actions menu""" diff --git a/datasette/views/database.py b/datasette/views/database.py index 58168ed7..53bdceed 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -110,6 +110,7 @@ class DatabaseView(DataView): datasette=self.ds, database=database, actor=request.actor, + request=request, ): extra_links = await await_me_maybe(hook) if extra_links: diff --git a/datasette/views/table.py b/datasette/views/table.py index b51d5e5e..81d4d721 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -894,6 +894,7 @@ class TableView(RowTableShared): table=table, database=database, actor=request.actor, + request=request, ): extra_links = await await_me_maybe(hook) if extra_links: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 688eaa61..2c31e6f4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1015,8 +1015,8 @@ The function can alternatively return an awaitable function if it needs to make .. _plugin_hook_menu_links: -menu_links(datasette, actor) ----------------------------- +menu_links(datasette, actor, request) +------------------------------------- ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. @@ -1024,6 +1024,9 @@ menu_links(datasette, actor) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. +``request`` - object or None + The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. + This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. The hook should return a list of ``{"href": "...", "label": "..."}`` menu items. These will be added to the menu. @@ -1045,11 +1048,10 @@ This example adds a new menu item but only if the signed in user is ``"root"``: Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. - .. _plugin_hook_table_actions: -table_actions(datasette, actor, database, table) ------------------------------------------------- +table_actions(datasette, actor, database, table, request) +--------------------------------------------------------- ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. @@ -1063,6 +1065,9 @@ table_actions(datasette, actor, database, table) ``table`` - string The name of the table. +``request`` - object + The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. + This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. It can alternatively return an ``async def`` awaitable function which returns a list of menu items. @@ -1083,8 +1088,8 @@ This example adds a new table action if the signed in user is ``"root"``: .. _plugin_hook_database_actions: -database_actions(datasette, actor, database) --------------------------------------------- +database_actions(datasette, actor, database, request) +----------------------------------------------------- ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. @@ -1095,4 +1100,7 @@ database_actions(datasette, actor, database) ``database`` - string The name of the database. +``request`` - object + The current HTTP :ref:`internals_request`. + This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 26d06091..85a7467d 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -316,9 +316,12 @@ def forbidden(datasette, request, message): @hookimpl -def menu_links(datasette, actor): +def menu_links(datasette, actor, request): if actor: - return [{"href": datasette.urls.instance(), "label": "Hello"}] + label = "Hello" + if request.args.get("_hello"): + label += ", " + request.args["_hello"] + return [{"href": datasette.urls.instance(), "label": label}] @hookimpl @@ -334,11 +337,14 @@ def table_actions(datasette, database, table, actor): @hookimpl -def database_actions(datasette, database, actor): +def database_actions(datasette, database, actor, request): if actor: + label = f"Database: {database}" + if request.args.get("_hello"): + label += " - " + request.args["_hello"] return [ { "href": datasette.urls.instance(), - "label": f"Database: {database}", + "label": label, } ] diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index f3b794cf..b70372f3 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -158,9 +158,12 @@ def menu_links(datasette, actor): @hookimpl -def table_actions(datasette, database, table, actor): +def table_actions(datasette, database, table, actor, request): async def inner(): if actor: - return [{"href": datasette.urls.instance(), "label": "From async"}] + label = "From async" + if request.args.get("_hello"): + label += " " + request.args["_hello"] + return [{"href": datasette.urls.instance(), "label": label}] return inner diff --git a/tests/test_plugins.py b/tests/test_plugins.py index ee6f1efa..b3561dd5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -781,9 +781,9 @@ def test_hook_menu_links(app_client): response = app_client.get("/") assert get_menu_links(response.text) == [] - response_2 = app_client.get("/?_bot=1") + response_2 = app_client.get("/?_bot=1&_hello=BOB") assert get_menu_links(response_2.text) == [ - {"label": "Hello", "href": "/"}, + {"label": "Hello, BOB", "href": "/"}, {"label": "Hello 2", "href": "/"}, ] @@ -800,12 +800,12 @@ def test_hook_table_actions(app_client, table_or_view): response = app_client.get(f"/fixtures/{table_or_view}") assert get_table_actions_links(response.text) == [] - response_2 = app_client.get(f"/fixtures/{table_or_view}?_bot=1") + response_2 = app_client.get(f"/fixtures/{table_or_view}?_bot=1&_hello=BOB") assert sorted( get_table_actions_links(response_2.text), key=lambda l: l["label"] ) == [ {"label": "Database: fixtures", "href": "/"}, - {"label": "From async", "href": "/"}, + {"label": "From async BOB", "href": "/"}, {"label": f"Table: {table_or_view}", "href": "/"}, ] @@ -821,7 +821,7 @@ def test_hook_database_actions(app_client): response = app_client.get("/fixtures") assert get_table_actions_links(response.text) == [] - response_2 = app_client.get("/fixtures?_bot=1") + response_2 = app_client.get("/fixtures?_bot=1&_hello=BOB") assert get_table_actions_links(response_2.text) == [ - {"label": "Database: fixtures", "href": "/"}, + {"label": "Database: fixtures - BOB", "href": "/"}, ] From cd7678fde65319d7b6955ce9f4678ba4b9e64b66 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Jun 2021 21:51:14 -0700 Subject: [PATCH 0366/1866] Release 0.58a0 Refs #1371 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 14a7be17..a46b4706 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.57.1" +__version__ = "0.58a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 89b8fcf5..99fc5ea5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_58a0: + +0.58a0 (2021-06-09) +------------------- + +- The :ref:`menu_links() `, :ref:`table_actions() ` and :ref:`database_actions() ` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`) + .. _v0_57_1: 0.57.1 (2021-06-08) From e7975657656ce02717f03703bb8ec17f2fe9b717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 13 Jun 2021 08:33:22 -0700 Subject: [PATCH 0367/1866] Bump black from 21.5b2 to 21.6b0 (#1374) Bumps [black](https://github.com/psf/black) from 21.5b2 to 21.6b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e66fefc3..767148ea 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ setup( "pytest-xdist>=2.2.1,<2.3", "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", - "black==21.5b2", + "black==21.6b0", "pytest-timeout>=1.4.2,<1.5", "trustme>=0.7,<0.8", ], From 83e9c8bc7585dcc62f200e37c2daefcd669ee05e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 13 Jun 2021 08:38:47 -0700 Subject: [PATCH 0368/1866] Update trustme requirement from <0.8,>=0.7 to >=0.7,<0.9 (#1373) Updates the requirements on [trustme](https://github.com/python-trio/trustme) to permit the latest version. - [Release notes](https://github.com/python-trio/trustme/releases) - [Commits](https://github.com/python-trio/trustme/compare/v0.7.0...v0.8.0) --- updated-dependencies: - dependency-name: trustme dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 767148ea..e9d4c8d1 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ setup( "beautifulsoup4>=4.8.1,<4.10.0", "black==21.6b0", "pytest-timeout>=1.4.2,<1.5", - "trustme>=0.7,<0.8", + "trustme>=0.7,<0.9", ], }, tests_require=["datasette[test]"], From 5335f360f4d57d70cab3694b08f15729c4ca2fe2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Jun 2021 17:17:06 -0700 Subject: [PATCH 0369/1866] Update pytest-xdist requirement from <2.3,>=2.2.1 to >=2.2.1,<2.4 (#1378) Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.2.1...v2.3.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e9d4c8d1..4f095f29 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], "test": [ "pytest>=5.2.2,<6.3.0", - "pytest-xdist>=2.2.1,<2.3", + "pytest-xdist>=2.2.1,<2.4", "pytest-asyncio>=0.10,<0.16", "beautifulsoup4>=4.8.1,<4.10.0", "black==21.6b0", From a6c55afe8c82ead8deb32f90c9324022fd422324 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Mon, 21 Jun 2021 11:57:38 -0400 Subject: [PATCH 0370/1866] Ensure db.path is a string before trying to insert into internal database (#1370) Thanks, @eyeseast --- datasette/app.py | 2 +- tests/test_api.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index fc5b7d9d..ce59ef54 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -354,7 +354,7 @@ class Datasette: INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) VALUES (?, ?, ?, ?) """, - [database_name, db.path, db.is_memory, schema_version], + [database_name, str(db.path), db.is_memory, schema_version], block=True, ) await populate_schema_tables(internal_db, db) diff --git a/tests/test_api.py b/tests/test_api.py index e5e609d6..2d891aae 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -25,6 +25,7 @@ from .fixtures import ( # noqa METADATA, ) import json +import pathlib import pytest import sys import urllib @@ -2123,3 +2124,16 @@ def test_col_nocol_errors(app_client, path, expected_error): response = app_client.get(path) assert response.status == 400 assert response.json["error"] == expected_error + + +@pytest.mark.asyncio +async def test_db_path(app_client): + db = app_client.ds.get_database() + path = pathlib.Path(db.path) + + assert path.exists() + + datasette = Datasette([path]) + + # this will break with a path + await datasette.refresh_schemas() From 7bc85b26d6b9c865caf949ff4660d855526c346e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 12:30:03 -0700 Subject: [PATCH 0371/1866] Deploy stable-docs.datasette.io on publish Refs https://github.com/simonw/datasette.io/issues/67 --- .github/workflows/publish.yml | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 90fa4505..8e4c2d02 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,6 +29,7 @@ jobs: - name: Run tests run: | pytest + deploy: runs-on: ubuntu-latest needs: [test] @@ -55,6 +56,47 @@ jobs: run: | python setup.py sdist bdist_wheel twine upload dist/* + + deploy_static_docs: + runs-on: ubuntu-latest + needs: [deploy] + # if: "!github.event.release.prerelease" + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-publish-pip- + - name: Install dependencies + run: | + python -m pip install -e .[docs] + python -m pip install sphinx-to-sqlite==0.1a1 + - name: Build docs.db + run: |- + cd docs + sphinx-build -b xml . _build + sphinx-to-sqlite ../docs.db _build + cd .. + - name: Set up Cloud Run + uses: google-github-actions/setup-gcloud@master + with: + version: '275.0.0' + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - name: Deploy stable-docs.datasette.io to Cloud Run + run: |- + gcloud config set run/region us-central1 + gcloud config set project datasette-222320 + datasette publish cloudrun docs.db \ + --service=datasette-docs-stable + deploy_docker: runs-on: ubuntu-latest needs: [deploy] From 403e370e5a3649333812edbbcba8467e6134cc16 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 12:50:19 -0700 Subject: [PATCH 0372/1866] Fixed reference to default publish implementation --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 2c31e6f4..331f8061 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -302,7 +302,7 @@ publish_subcommand(publish) The Click command group for the ``datasette publish`` subcommand This hook allows you to create new providers for the ``datasette publish`` -command. Datasette uses this hook internally to implement the default ``now`` +command. Datasette uses this hook internally to implement the default ``cloudrun`` and ``heroku`` subcommands, so you can read `their source `_ to see examples of this hook in action. From 3a500155663a07720a8a7baa04acda8c4c937692 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 12:51:19 -0700 Subject: [PATCH 0373/1866] datasette-publish-now is now called datasette-publish-vercel --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 331f8061..8b2a691a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -348,7 +348,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ ): # Your implementation goes here -Examples: `datasette-publish-fly `_, `datasette-publish-now `_ +Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ .. _plugin_hook_render_cell: From 4a3e8561ab109f3f171726bc2a7ebac1f23b72a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 15:27:30 -0700 Subject: [PATCH 0374/1866] Default 405 for POST, plus tests --- datasette/views/base.py | 3 +++ tests/test_html.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/datasette/views/base.py b/datasette/views/base.py index 1a03b97f..a87a0e77 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -106,6 +106,9 @@ class BaseView: async def options(self, request, *args, **kwargs): return Response.text("Method not allowed", status=405) + async def post(self, request, *args, **kwargs): + return Response.text("Method not allowed", status=405) + async def put(self, request, *args, **kwargs): return Response.text("Method not allowed", status=405) diff --git a/tests/test_html.py b/tests/test_html.py index ccee8b7e..aee6bce1 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -92,6 +92,13 @@ def test_memory_database_page(): assert response.status == 200 +def test_not_allowed_methods(): + with make_app_client(memory=True) as client: + for method in ("post", "put", "patch", "delete"): + response = client.request(path="/_memory", method=method.upper()) + assert response.status == 405 + + def test_database_page_redirects_with_url_hash(app_client_with_hash): response = app_client_with_hash.get("/fixtures", allow_redirects=False) assert response.status == 302 From b1fd24ac9f9035464af0a8ce92391c166a783253 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 15:39:52 -0700 Subject: [PATCH 0375/1866] skip_csrf(datasette, scope) plugin hook, refs #1377 --- datasette/app.py | 3 +++ datasette/hookspecs.py | 5 +++++ docs/internals.rst | 2 ++ docs/plugin_hooks.rst | 25 +++++++++++++++++++++++++ setup.py | 2 +- tests/fixtures.py | 2 ++ tests/plugins/my_plugin.py | 5 +++++ tests/test_plugins.py | 25 +++++++++++++++++++++++++ 8 files changed, 68 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index ce59ef54..e11c12eb 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1052,6 +1052,9 @@ class Datasette: DatasetteRouter(self, routes), signing_secret=self._secret, cookie_name="ds_csrftoken", + skip_if_scope=lambda scope: any( + pm.hook.skip_csrf(datasette=self, scope=scope) + ), ) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 579787a2..63b06097 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -112,3 +112,8 @@ def table_actions(datasette, actor, database, table, request): @hookspec def database_actions(datasette, actor, database, request): """Links for the database actions menu""" + + +@hookspec +def skip_csrf(datasette, scope): + """Mechanism for skipping CSRF checks for certain requests""" diff --git a/docs/internals.rst b/docs/internals.rst index 72c86083..98df998a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -778,6 +778,8 @@ If your plugin implements a ```` anywhere you will need to i +You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csrf` hook. + .. _internals_internal: The _internal database diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 8b2a691a..5af601b4 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1104,3 +1104,28 @@ database_actions(datasette, actor, database, request) The current HTTP :ref:`internals_request`. This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. + +.. _plugin_hook_skip_csrf: + +skip_csrf(datasette, scope) +--------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``scope`` - dictionary + The `ASGI scope `__ for the incoming HTTP request. + +This hook can be used to skip :ref:`internals_csrf` for a specific incoming request. For example, you might have a custom path at ``/submit-comment`` which is designed to accept comments from anywhere, whether or not the incoming request originated on the site and has an accompanying CSRF token. + +This example will disable CSRF protection for that specific URL path: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def skip_csrf(scope): + return scope["path"] == "/submit-comment" + +If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request. diff --git a/setup.py b/setup.py index 4f095f29..8a651d32 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ setup( "uvicorn~=0.11", "aiofiles>=0.4,<0.8", "janus>=0.4,<0.7", - "asgi-csrf>=0.6", + "asgi-csrf>=0.9", "PyYAML~=5.3", "mergedeep>=1.1.1,<1.4.0", "itsdangerous>=1.1,<3.0", diff --git a/tests/fixtures.py b/tests/fixtures.py index cdd2e987..a79fc246 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -52,6 +52,7 @@ EXPECTED_PLUGINS = [ "register_magic_parameters", "register_routes", "render_cell", + "skip_csrf", "startup", "table_actions", ], @@ -152,6 +153,7 @@ def make_app_client( static_mounts=static_mounts, template_dir=template_dir, crossdb=crossdb, + pdb=True, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) yield TestClient(ds) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 85a7467d..0e625623 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -348,3 +348,8 @@ def database_actions(datasette, database, actor, request): "label": label, } ] + + +@hookimpl +def skip_csrf(scope): + return scope["path"] == "/skip-csrf" diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b3561dd5..14273282 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -825,3 +825,28 @@ def test_hook_database_actions(app_client): assert get_table_actions_links(response_2.text) == [ {"label": "Database: fixtures - BOB", "href": "/"}, ] + + +def test_hook_skip_csrf(app_client): + cookie = app_client.actor_cookie({"id": "test"}) + csrf_response = app_client.post( + "/post/", + post_data={"this is": "post data"}, + csrftoken_from=True, + cookies={"ds_actor": cookie}, + ) + assert csrf_response.status == 200 + missing_csrf_response = app_client.post( + "/post/", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} + ) + assert missing_csrf_response.status == 403 + # But "/skip-csrf" should allow + allow_csrf_response = app_client.post( + "/skip-csrf", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} + ) + assert allow_csrf_response.status == 405 # Method not allowed + # /skip-csrf-2 should not + second_missing_csrf_response = app_client.post( + "/skip-csrf-2", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} + ) + assert second_missing_csrf_response.status == 403 From 02b19c7a9afd328f22040ab33b5c1911cd904c7c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Jun 2021 15:50:48 -0700 Subject: [PATCH 0376/1866] Removed rogue pdb=True, refs #1377 --- tests/fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index a79fc246..1fb52bf9 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -153,7 +153,6 @@ def make_app_client( static_mounts=static_mounts, template_dir=template_dir, crossdb=crossdb, - pdb=True, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) yield TestClient(ds) From ff17970ed4988a80b699d417bbeec07d63400e24 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 09:24:59 -0700 Subject: [PATCH 0377/1866] Release 0.58a1 Refs #1365, #1377 --- datasette/version.py | 2 +- docs/changelog.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index a46b4706..e5a29931 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.58a0" +__version__ = "0.58a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 99fc5ea5..bcd8b987 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog ========= +.. _v0_58a1: + +0.58a1 (2021-06-24) +------------------- + +- New plugin hook: :ref:`plugin_hook_skip_csrf`, for opting out of CSRF protection based on the incoming request. (:issue:`1377`) +- ``POST`` requests to endpoints that do not support that HTTP verb now return a 405 error. +- ``db.path`` can now be provided as a ``pathlib.Path`` object, useful when writing unit tests for plugins. Thanks, Chris Amico. (:issue:`1365`) + .. _v0_58a0: 0.58a0 (2021-06-09) From 953a64467d78bca29fe6cc18bdb2baa7848e53ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Jun 2021 09:42:02 -0700 Subject: [PATCH 0378/1866] Only publish stable docs on non-preview release Refs https://github.com/simonw/datasette.io/issues/67 --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e4c2d02..727f9933 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -60,7 +60,7 @@ jobs: deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] - # if: "!github.event.release.prerelease" + if: "!github.event.release.prerelease" steps: - uses: actions/checkout@v2 - name: Set up Python From baf986c871708c01ca183be760995cf306ba21bf Mon Sep 17 00:00:00 2001 From: Brandon Roberts Date: Sat, 26 Jun 2021 15:24:54 -0700 Subject: [PATCH 0379/1866] New get_metadata() plugin hook for dynamic metadata The following hook is added: get_metadata( datasette=self, key=key, database=database, table=table, fallback=fallback ) This gets called when we're building our metdata for the rest of the system to use. We merge whatever the plugins return with any local metadata (from metadata.yml/yaml/json) allowing for a live-editable dynamic Datasette. As a security precation, local meta is *not* overwritable by plugin hooks. The workflow for transitioning to live-meta would be to load the plugin with the full metadata.yaml and save. Then remove the parts of the metadata that you want to be able to change from the file. * Avoid race condition: don't mutate databases list This avoids the nasty "RuntimeError: OrderedDict mutated during iteration" error that randomly happens when a plugin adds a new database to Datasette, using `add_database`. This change makes the add and remove database functions more expensive, but it prevents the random explosion race conditions that make for confusing user experience when importing live databases. Thanks, @brandonrobertz --- .gitignore | 1 + datasette/app.py | 47 ++++++++++++++++++++++++++++++++----- datasette/hookspecs.py | 5 ++++ datasette/utils/__init__.py | 1 - docs/plugin_hooks.rst | 35 +++++++++++++++++++++++++++ tests/test_permissions.py | 6 ++--- tests/test_plugins.py | 29 +++++++++++++++++++++++ 7 files changed, 114 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 29ac176f..066009f0 100644 --- a/.gitignore +++ b/.gitignore @@ -117,3 +117,4 @@ ENV/ # macOS files .DS_Store node_modules +.*.swp diff --git a/datasette/app.py b/datasette/app.py index e11c12eb..05ad5a8d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -251,7 +251,7 @@ class Datasette: if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) - self._metadata = metadata or {} + self._metadata_local = metadata or {} self.sqlite_functions = [] self.sqlite_extensions = [] for extension in sqlite_extensions or []: @@ -380,6 +380,7 @@ class Datasette: return self.databases[name] def add_database(self, db, name=None): + new_databases = self.databases.copy() if name is None: # Pick a unique name for this database suggestion = db.suggest_name() @@ -391,14 +392,18 @@ class Datasette: name = "{}_{}".format(suggestion, i) i += 1 db.name = name - self.databases[name] = db + new_databases[name] = db + # don't mutate! that causes race conditions with live import + self.databases = new_databases return db def add_memory_database(self, memory_name): return self.add_database(Database(self, memory_name=memory_name)) def remove_database(self, name): - self.databases.pop(name) + new_databases = self.databases.copy() + new_databases.pop(name) + self.databases = new_databases def setting(self, key): return self._settings.get(key, None) @@ -407,6 +412,17 @@ class Datasette: # Returns a fully resolved config dictionary, useful for templates return {option.name: self.setting(option.name) for option in SETTINGS} + def _metadata_recursive_update(self, orig, updated): + if not isinstance(orig, dict) or not isinstance(updated, dict): + return orig + + for key, upd_value in updated.items(): + if isinstance(upd_value, dict) and isinstance(orig.get(key), dict): + orig[key] = self._metadata_recursive_update(orig[key], upd_value) + else: + orig[key] = upd_value + return orig + def metadata(self, key=None, database=None, table=None, fallback=True): """ Looks up metadata, cascading backwards from specified level. @@ -415,7 +431,21 @@ class Datasette: assert not ( database is None and table is not None ), "Cannot call metadata() with table= specified but not database=" - databases = self._metadata.get("databases") or {} + metadata = {} + + for hook_dbs in pm.hook.get_metadata( + datasette=self, key=key, database=database, table=table, fallback=fallback + ): + metadata = self._metadata_recursive_update(metadata, hook_dbs) + + # security precaution!! don't allow anything in the local config + # to be overwritten. this is a temporary measure, not sure if this + # is a good idea long term or maybe if it should just be a concern + # of the plugin's implemtnation + metadata = self._metadata_recursive_update(metadata, self._metadata_local) + + databases = metadata.get("databases") or {} + search_list = [] if database is not None: search_list.append(databases.get(database) or {}) @@ -424,7 +454,8 @@ class Datasette: table ) or {} search_list.insert(0, table_metadata) - search_list.append(self._metadata) + + search_list.append(metadata) if not fallback: # No fallback allowed, so just use the first one in the list search_list = search_list[:1] @@ -440,6 +471,10 @@ class Datasette: m.update(item) return m + @property + def _metadata(self): + return self.metadata() + def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( @@ -960,7 +995,7 @@ class Datasette: r"/:memory:(?P.*)$", ) add_route( - JsonDataView.as_view(self, "metadata.json", lambda: self._metadata), + JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()), r"/-/metadata(?P(\.json)?)$", ) add_route( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 63b06097..c40b3148 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -10,6 +10,11 @@ def startup(datasette): """Fires directly after Datasette first starts running""" +@hookspec +def get_metadata(datasette, key, database, table, fallback): + """Get configuration""" + + @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 73122976..1e193862 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -21,7 +21,6 @@ import numbers import yaml from .shutil_backport import copytree from .sqlite import sqlite3, sqlite_version, supports_table_xinfo -from ..plugins import pm # From https://www.sqlite.org/lang_keywords.html diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 5af601b4..9ec75f34 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1129,3 +1129,38 @@ This example will disable CSRF protection for that specific URL path: return scope["path"] == "/submit-comment" If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request. + +get_metadata(datasette, key, database, table, fallback) +------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string or None + The name of the database metadata is being asked for. + +``table`` - string or None + The name of the table. + +``key`` - string or None + The name of the key for which data is being asked for. + +This hook is responsible for returning a dictionary corresponding to Datasette :ref:`metadata`. This function is passed the `database`, `table` and `key` which were passed to the upstream internal request for metadata. Regardless, it is important to return a global metadata object, where `"databases": []` would be a top-level key. The dictionary returned here, will be merged with, and overwritten by, the contents of the physical `metadata.yaml` if one is present. + +.. code-block:: python + + @hookimpl + def get_metadata(datasette, key, database, table, fallback): + metadata = { + "title": "This will be the Datasette landing page title!", + "description": get_instance_description(datasette), + "databases": [], + } + for db_name, db_data_dict in get_my_database_meta(datasette, database, table, key): + metadata["databases"][db_name] = db_data_dict + # whatever we return here will be merged with any other plugins using this hook and + # will be overwritten by a local metadata.yaml if one exists! + return metadata diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 9317c0d9..788523b0 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -440,7 +440,7 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta """Test that e.g. having view-table but NOT view-database lets you view table page, etc""" allow = {"id": "*"} deny = {} - previous_metadata = cascade_app_client.ds._metadata + previous_metadata = cascade_app_client.ds.metadata() updated_metadata = copy.deepcopy(previous_metadata) actor = {"id": "test"} if "download" in permissions: @@ -457,11 +457,11 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta updated_metadata["databases"]["fixtures"]["queries"]["magic_parameters"][ "allow" ] = (allow if "query" in permissions else deny) - cascade_app_client.ds._metadata = updated_metadata + cascade_app_client.ds._metadata_local = updated_metadata response = cascade_app_client.get( path, cookies={"ds_actor": cascade_app_client.actor_cookie(actor)}, ) assert expected_status == response.status finally: - cascade_app_client.ds._metadata = previous_metadata + cascade_app_client.ds._metadata_local = previous_metadata diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 14273282..3b9c06b9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -850,3 +850,32 @@ def test_hook_skip_csrf(app_client): "/skip-csrf-2", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} ) assert second_missing_csrf_response.status == 403 + + +def test_hook_get_metadata(app_client): + app_client.ds._metadata_local = { + "title": "Testing get_metadata hook!", + "databases": { + "from-local": { + "title": "Hello from local metadata" + } + } + } + og_pm_hook_get_metadata = pm.hook.get_metadata + def get_metadata_mock(*args, **kwargs): + return [{ + "databases": { + "from-hook": { + "title": "Hello from the plugin hook" + }, + "from-local": { + "title": "This will be overwritten!" + } + } + }] + pm.hook.get_metadata = get_metadata_mock + meta = app_client.ds.metadata() + assert "Testing get_metadata hook!" == meta["title"] + assert "Hello from local metadata" == meta["databases"]["from-local"]["title"] + assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"] + pm.hook.get_metadata = og_pm_hook_get_metadata From 05a312caf3debb51aa1069939923a49e21cd2bd1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 26 Jun 2021 15:25:28 -0700 Subject: [PATCH 0380/1866] Applied Black, refs #1368 --- tests/test_plugins.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 3b9c06b9..7a626ce5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -855,24 +855,20 @@ def test_hook_skip_csrf(app_client): def test_hook_get_metadata(app_client): app_client.ds._metadata_local = { "title": "Testing get_metadata hook!", - "databases": { - "from-local": { - "title": "Hello from local metadata" - } - } + "databases": {"from-local": {"title": "Hello from local metadata"}}, } og_pm_hook_get_metadata = pm.hook.get_metadata + def get_metadata_mock(*args, **kwargs): - return [{ - "databases": { - "from-hook": { - "title": "Hello from the plugin hook" - }, - "from-local": { - "title": "This will be overwritten!" + return [ + { + "databases": { + "from-hook": {"title": "Hello from the plugin hook"}, + "from-local": {"title": "This will be overwritten!"}, } } - }] + ] + pm.hook.get_metadata = get_metadata_mock meta = app_client.ds.metadata() assert "Testing get_metadata hook!" == meta["title"] From 089278b8dbe0cb3d41f27666d97b0096b750fbe2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 26 Jun 2021 15:49:07 -0700 Subject: [PATCH 0381/1866] rST fix, refs #1384 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9ec75f34..d3b55747 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1148,7 +1148,7 @@ get_metadata(datasette, key, database, table, fallback) ``key`` - string or None The name of the key for which data is being asked for. -This hook is responsible for returning a dictionary corresponding to Datasette :ref:`metadata`. This function is passed the `database`, `table` and `key` which were passed to the upstream internal request for metadata. Regardless, it is important to return a global metadata object, where `"databases": []` would be a top-level key. The dictionary returned here, will be merged with, and overwritten by, the contents of the physical `metadata.yaml` if one is present. +This hook is responsible for returning a dictionary corresponding to Datasette :ref:`metadata`. This function is passed the ``database``, ``table`` and ``key`` which were passed to the upstream internal request for metadata. Regardless, it is important to return a global metadata object, where ``"databases": []`` would be a top-level key. The dictionary returned here, will be merged with, and overwritten by, the contents of the physical ``metadata.yaml`` if one is present. .. code-block:: python From 0d339a4897c808903e34fa6be228cdaaa5a29c55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 26 Jun 2021 16:04:39 -0700 Subject: [PATCH 0382/1866] Removed text about executing SQL, refs #1384 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d3b55747..d71037d9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1134,7 +1134,7 @@ get_metadata(datasette, key, database, table, fallback) ------------------------------------------------------- ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. ``actor`` - dictionary or None The currently authenticated :ref:`actor `. From ea627baccf980d7d8ebc9e1ffff1fe34d556e56f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 26 Jun 2021 17:02:42 -0700 Subject: [PATCH 0383/1866] Removed fallback parameter from get_metadata, refs #1384 --- datasette/app.py | 2 +- datasette/hookspecs.py | 4 ++-- docs/plugin_hooks.rst | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 05ad5a8d..0b909968 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -434,7 +434,7 @@ class Datasette: metadata = {} for hook_dbs in pm.hook.get_metadata( - datasette=self, key=key, database=database, table=table, fallback=fallback + datasette=self, key=key, database=database, table=table ): metadata = self._metadata_recursive_update(metadata, hook_dbs) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index c40b3148..07b2f5ba 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -11,8 +11,8 @@ def startup(datasette): @hookspec -def get_metadata(datasette, key, database, table, fallback): - """Get configuration""" +def get_metadata(datasette, key, database, table): + """Return metadata to be merged into Datasette's metadata dictionary""" @hookspec diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d71037d9..b687a6e7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1130,8 +1130,8 @@ This example will disable CSRF protection for that specific URL path: If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request. -get_metadata(datasette, key, database, table, fallback) -------------------------------------------------------- +get_metadata(datasette, key, database, table) +--------------------------------------------- ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. @@ -1153,7 +1153,7 @@ This hook is responsible for returning a dictionary corresponding to Datasette : .. code-block:: python @hookimpl - def get_metadata(datasette, key, database, table, fallback): + def get_metadata(datasette, key, database, table): metadata = { "title": "This will be the Datasette landing page title!", "description": get_instance_description(datasette), From dbc61a1fd343e4660b6220f60c4ce79341245048 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 2 Jul 2021 10:33:03 -0700 Subject: [PATCH 0384/1866] Documented ProxyPreserveHost On for Apache, closes #1387 --- docs/deploying.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 48261b59..47dff73d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -161,6 +161,9 @@ For `Apache `__, you can use the ``ProxyPass`` direct LoadModule proxy_module lib/httpd/modules/mod_proxy.so LoadModule proxy_http_module lib/httpd/modules/mod_proxy_http.so -Then add this directive to proxy traffic:: +Then add these directives to proxy traffic:: - ProxyPass /datasette-prefix/ http://127.0.0.1:8009/datasette-prefix/ + ProxyPass /datasette-prefix/ http://127.0.0.1:8009/datasette-prefix/ + ProxyPreserveHost On + +The `ProxyPreserveHost On `__ directive ensures that the original ``Host:`` header from the incoming request is passed through to Datasette. Datasette needs this to correctly assemble links to other pages using the :ref:`datasette_absolute_url` method. From c8feaf0b628ddb1f98b2a4b89691d3d1b939ed8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Jul 2021 09:32:32 -0700 Subject: [PATCH 0385/1866] systemctl restart datasette.service, closes #1390 --- docs/deploying.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 47dff73d..44ddd07b 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -65,7 +65,11 @@ You can start the Datasette process running using the following:: sudo systemctl daemon-reload sudo systemctl start datasette.service -You can confirm that Datasette is running on port 8000 like so:: +You may need to restart the Datasette service after making changes to its ``metadata.json`` configuration or the ``datasette.service`` file. You can do that using:: + + sudo systemctl restart datasette.service + +Once the service has started you can confirm that Datasette is running on port 8000 like so:: curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version From 83f6799a96f48b5acef4911c0273973f15efdf05 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 11:30:48 -0700 Subject: [PATCH 0386/1866] searchmode: raw table metadata property, closes #1389 --- datasette/views/table.py | 8 +++++++- docs/full_text_search.rst | 29 ++++++++++++++++++---------- tests/test_api.py | 40 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 81d4d721..1bda7496 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -495,7 +495,13 @@ class TableView(RowTableShared): if pair[0].startswith("_search") and pair[0] != "_searchmode" ) search = "" - search_mode_raw = special_args.get("_searchmode") == "raw" + search_mode_raw = table_metadata.get("searchmode") == "raw" + # Or set it from the querystring + qs_searchmode = special_args.get("_searchmode") + if qs_searchmode == "escaped": + search_mode_raw = False + if qs_searchmode == "raw": + search_mode_raw = True if fts_table and search_args: if "_search" in search_args: # Simple ?_search=xxx diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index b414ff37..f549296f 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -36,7 +36,11 @@ Advanced SQLite search queries SQLite full-text search includes support for `a variety of advanced queries `__, including ``AND``, ``OR``, ``NOT`` and ``NEAR``. -By default Datasette disables these features to ensure they do not cause any confusion for users who are not aware of them. You can disable this escaping and use the advanced queries by adding ``?_searchmode=raw`` to the table page query string. +By default Datasette disables these features to ensure they do not cause errors or confusion for users who are not aware of them. You can disable this escaping and use the advanced queries by adding ``&_searchmode=raw`` to the table page query string. + +If you want to enable these operators by default for a specific table, you can do so by adding ``"searchmode": "raw"`` to the metadata configuration for that table, see :ref:`full_text_search_table_or_view`. + +If that option has been specified in the table metadata but you want to over-ride it and return to the default behavior you can append ``&_searchmode=escaped`` to the query string. .. _full_text_search_table_or_view: @@ -53,19 +57,24 @@ https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_ The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property. -Here is an example which enables full-text search for a ``display_ads`` view which is defined against the ``ads`` table and hence needs to run FTS against the ``ads_fts`` table, using the ``id`` as the primary key:: +The ``"searchmode": "raw"`` property can be used to default the table to accepting SQLite advanced search operators, as described in :ref:`full_text_search_advanced_queries`. + +Here is an example which enables full-text search (with SQLite advanced search operators) for a ``display_ads`` view which is defined against the ``ads`` table and hence needs to run FTS against the ``ads_fts`` table, using the ``id`` as the primary key: + +.. code-block:: json { - "databases": { - "russian-ads": { - "tables": { - "display_ads": { - "fts_table": "ads_fts", - "fts_pk": "id" + "databases": { + "russian-ads": { + "tables": { + "display_ads": { + "fts_table": "ads_fts", + "fts_pk": "id", + "search_mode": "raw" + } + } } - } } - } } .. _full_text_search_custom_sql: diff --git a/tests/test_api.py b/tests/test_api.py index 2d891aae..cb3c255d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1078,6 +1078,46 @@ def test_searchable(app_client, path, expected_rows): assert expected_rows == response.json["rows"] +_SEARCHMODE_RAW_RESULTS = [ + [1, "barry cat", "terry dog", "panther"], + [2, "terry dog", "sara weasel", "puma"], +] + + +@pytest.mark.parametrize( + "table_metadata,querystring,expected_rows", + [ + ( + {}, + "_search=te*+AND+do*", + [], + ), + ( + {"searchmode": "raw"}, + "_search=te*+AND+do*", + _SEARCHMODE_RAW_RESULTS, + ), + ( + {}, + "_search=te*+AND+do*&_searchmode=raw", + _SEARCHMODE_RAW_RESULTS, + ), + # Can be over-ridden with _searchmode=escaped + ( + {"searchmode": "raw"}, + "_search=te*+AND+do*&_searchmode=escaped", + [], + ), + ], +) +def test_searchmode(table_metadata, querystring, expected_rows): + with make_app_client( + metadata={"databases": {"fixtures": {"tables": {"searchable": table_metadata}}}} + ) as client: + response = client.get("/fixtures/searchable.json?" + querystring) + assert expected_rows == response.json["rows"] + + @pytest.mark.parametrize( "path,expected_rows", [ From 2e8d924cdc2274eb31fb76332bc5269f65c0ad90 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 12:03:19 -0700 Subject: [PATCH 0387/1866] Refactored generated_columns test, no longer in fixtures.db - refs #1391 --- tests/fixtures.py | 19 +-------- tests/test_api.py | 52 ++++++++++++------------ tests/test_internals_database.py | 70 +++++++++++++++----------------- 3 files changed, 59 insertions(+), 82 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 1fb52bf9..dce94876 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,5 +1,5 @@ from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3, sqlite_version, supports_generated_columns +from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils.testing import TestClient import click import contextlib @@ -118,8 +118,6 @@ def make_app_client( immutables = [] conn = sqlite3.connect(filepath) conn.executescript(TABLES) - if supports_generated_columns(): - conn.executescript(GENERATED_COLUMNS_SQL) for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) @@ -720,18 +718,6 @@ INSERT INTO "searchable_fts" (rowid, text1, text2) SELECT rowid, text1, text2 FROM searchable; """ -GENERATED_COLUMNS_SQL = """ -CREATE TABLE generated_columns ( - body TEXT, - id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, - consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED -); -INSERT INTO generated_columns (body) VALUES ('{ - "number": 1, - "string": "This is a string" -}'); -""" - def assert_permissions_checked(datasette, actions): # actions is a list of "action" or (action, resource) tuples @@ -792,9 +778,6 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) - if supports_generated_columns(): - with conn: - conn.executescript(GENERATED_COLUMNS_SQL) print(f"Test tables written to {db_filename}") if metadata: with open(metadata, "w") as fp: diff --git a/tests/test_api.py b/tests/test_api.py index cb3c255d..3e8d02c8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,7 +20,6 @@ from .fixtures import ( # noqa generate_compound_rows, generate_sortable_rows, make_app_client, - supports_generated_columns, EXPECTED_PLUGINS, METADATA, ) @@ -38,7 +37,7 @@ def test_homepage(app_client): assert response.json.keys() == {"fixtures": 0}.keys() d = response.json["fixtures"] assert d["name"] == "fixtures" - assert d["tables_count"] == 25 if supports_generated_columns() else 24 + assert d["tables_count"] == 24 assert len(d["tables_and_views_truncated"]) == 5 assert d["tables_and_views_more"] is True # 4 hidden FTS tables + no_primary_key (hidden in metadata) @@ -271,22 +270,7 @@ def test_database_page(app_client): }, "private": False, }, - ] + ( - [ - { - "columns": ["body", "id", "consideration"], - "count": 1, - "foreign_keys": {"incoming": [], "outgoing": []}, - "fts_table": None, - "hidden": False, - "name": "generated_columns", - "primary_keys": [], - "private": False, - } - ] - if supports_generated_columns() - else [] - ) + [ + ] + [ { "name": "infinity", "columns": ["value"], @@ -2074,16 +2058,30 @@ def test_paginate_using_link_header(app_client, qs): sqlite_version() < (3, 31, 0), reason="generated columns were added in SQLite 3.31.0", ) -def test_generated_columns_are_visible_in_datasette(app_client): - response = app_client.get("/fixtures/generated_columns.json?_shape=array") - assert response.json == [ - { - "rowid": 1, - "body": '{\n "number": 1,\n "string": "This is a string"\n}', - "id": 1, - "consideration": "This is a string", +def test_generated_columns_are_visible_in_datasette(): + with make_app_client( + extra_databases={ + "generated.db": """ + CREATE TABLE generated_columns ( + body TEXT, + id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, + consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED + ); + INSERT INTO generated_columns (body) VALUES ('{ + "number": 1, + "string": "This is a string" + }');""" } - ] + ) as client: + response = app_client.get("/generated/generated_columns.json?_shape=array") + assert response.json == [ + { + "rowid": 1, + "body": '{\n "number": 1,\n "string": "This is a string"\n}', + "id": 1, + "consideration": "This is a string", + } + ] def test_http_options_request(app_client): diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index b60aaa8e..ad829751 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,7 +2,7 @@ Tests for the datasette.database.Database class """ from datasette.database import Database, Results, MultipleValues -from datasette.utils.sqlite import sqlite3, supports_generated_columns +from datasette.utils.sqlite import sqlite3 from datasette.utils import Column from .fixtures import app_client, app_client_two_attached_databases_crossdb_enabled import pytest @@ -340,42 +340,38 @@ async def test_get_all_foreign_keys(db): @pytest.mark.asyncio async def test_table_names(db): table_names = await db.table_names() - assert ( - table_names - == [ - "simple_primary_key", - "primary_key_multiple_columns", - "primary_key_multiple_columns_explicit_label", - "compound_primary_key", - "compound_three_primary_keys", - "foreign_key_references", - "sortable", - "no_primary_key", - "123_starts_with_digits", - "Table With Space In Name", - "table/with/slashes.csv", - "complex_foreign_keys", - "custom_foreign_key_label", - "units", - "tags", - "searchable", - "searchable_tags", - "searchable_fts", - "searchable_fts_segments", - "searchable_fts_segdir", - "searchable_fts_docsize", - "searchable_fts_stat", - "select", - "infinity", - "facet_cities", - "facetable", - "binary_data", - "roadside_attractions", - "attraction_characteristic", - "roadside_attraction_characteristics", - ] - + (["generated_columns"] if supports_generated_columns() else []) - ) + assert table_names == [ + "simple_primary_key", + "primary_key_multiple_columns", + "primary_key_multiple_columns_explicit_label", + "compound_primary_key", + "compound_three_primary_keys", + "foreign_key_references", + "sortable", + "no_primary_key", + "123_starts_with_digits", + "Table With Space In Name", + "table/with/slashes.csv", + "complex_foreign_keys", + "custom_foreign_key_label", + "units", + "tags", + "searchable", + "searchable_tags", + "searchable_fts", + "searchable_fts_segments", + "searchable_fts_segdir", + "searchable_fts_docsize", + "searchable_fts_stat", + "select", + "infinity", + "facet_cities", + "facetable", + "binary_data", + "roadside_attractions", + "attraction_characteristic", + "roadside_attraction_characteristics", + ] @pytest.mark.asyncio From e0064ba7b06973eae70e6222a6208d9fed5bd170 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 12:14:14 -0700 Subject: [PATCH 0388/1866] Fixes for test_generated_columns_are_visible_in_datasette, refs #1391 --- tests/test_api.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 3e8d02c8..0049d76d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2067,17 +2067,16 @@ def test_generated_columns_are_visible_in_datasette(): id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED, consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED ); - INSERT INTO generated_columns (body) VALUES ('{ - "number": 1, - "string": "This is a string" - }');""" + INSERT INTO generated_columns (body) VALUES ( + '{"number": 1, "string": "This is a string"}' + );""" } ) as client: - response = app_client.get("/generated/generated_columns.json?_shape=array") + response = client.get("/generated/generated_columns.json?_shape=array") assert response.json == [ { "rowid": 1, - "body": '{\n "number": 1,\n "string": "This is a string"\n}', + "body": '{"number": 1, "string": "This is a string"}', "id": 1, "consideration": "This is a string", } From 180c7a5328457aefdf847ada366e296fef4744f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 16:37:30 -0700 Subject: [PATCH 0389/1866] --uds option for binding to Unix domain socket, closes #1388 --- datasette/cli.py | 7 +++++++ docs/datasette-serve-help.txt | 1 + docs/deploying.rst | 23 ++++++++++++++++++++++- tests/conftest.py | 20 +++++++++++++++++++- tests/test_cli.py | 1 + tests/test_cli_serve_server.py | 15 +++++++++++++++ 6 files changed, 65 insertions(+), 2 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 12ee92c3..09aebcc8 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -333,6 +333,10 @@ def uninstall(packages, yes): type=click.IntRange(0, 65535), help="Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.", ) +@click.option( + "--uds", + help="Bind to a Unix domain socket", +) @click.option( "--reload", is_flag=True, @@ -428,6 +432,7 @@ def serve( immutable, host, port, + uds, reload, cors, sqlite_extensions, @@ -569,6 +574,8 @@ def serve( uvicorn_kwargs = dict( host=host, port=port, log_level="info", lifespan="on", workers=1 ) + if uds: + uvicorn_kwargs["uds"] = uds if ssl_keyfile: uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile if ssl_certfile: diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index db51dd80..ec3f41a0 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -12,6 +12,7 @@ Options: machines. -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to automatically assign an available port. [0<=x<=65535] + --uds TEXT Bind to a Unix domain socket --reload Automatically reload if code or metadata change detected - useful for development --cors Enable CORS by serving Access-Control-Allow-Origin: * diff --git a/docs/deploying.rst b/docs/deploying.rst index 44ddd07b..f3680034 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -148,7 +148,6 @@ Here is an example of an `nginx `__ configuration file that http { server { listen 80; - location /my-datasette { proxy_pass http://127.0.0.1:8009/my-datasette; proxy_set_header X-Real-IP $remote_addr; @@ -157,6 +156,28 @@ Here is an example of an `nginx `__ configuration file that } } +You can also use the ``--uds`` option to Datasette to listen on a Unix domain socket instead of a port, configuring the nginx upstream proxy like this:: + + daemon off; + events { + worker_connections 1024; + } + http { + server { + listen 80; + location / { + proxy_pass http://datasette; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + upstream datasette { + server unix:/tmp/datasette.sock; + } + } + +Then run Datasette with ``datasette --uds /tmp/datasette.sock path/to/database.db``. + Apache proxy configuration -------------------------- diff --git a/tests/conftest.py b/tests/conftest.py index c6a3eee6..34a64efc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -131,7 +131,6 @@ def ds_localhost_https_server(tmp_path_factory): for blob in server_cert.cert_chain_pems: blob.write_to_path(path=certfile, append=True) ca.cert_pem.write_to_path(path=client_cert) - ds_proc = subprocess.Popen( [ "datasette", @@ -154,3 +153,22 @@ def ds_localhost_https_server(tmp_path_factory): yield ds_proc, client_cert # Shut it down at the end of the pytest session ds_proc.terminate() + + +@pytest.fixture(scope="session") +def ds_unix_domain_socket_server(tmp_path_factory): + socket_folder = tmp_path_factory.mktemp("uds") + uds = str(socket_folder / "datasette.sock") + ds_proc = subprocess.Popen( + ["datasette", "--memory", "--uds", uds], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=tempfile.gettempdir(), + ) + # Give the server time to start + time.sleep(1.5) + # Check it started successfully + assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") + yield ds_proc, uds + # Shut it down at the end of the pytest session + ds_proc.terminate() diff --git a/tests/test_cli.py b/tests/test_cli.py index e094ccb6..e31a305e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -132,6 +132,7 @@ def test_metadata_yaml(): immutable=[], host="127.0.0.1", port=8001, + uds=None, reload=False, cors=False, sqlite_extensions=[], diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 6f5366d1..73439125 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,5 +1,6 @@ import httpx import pytest +import socket @pytest.mark.serial @@ -21,3 +22,17 @@ def test_serve_localhost_https(ds_localhost_https_server): "path": "/_memory", "tables": [], }.items() <= response.json().items() + + +@pytest.mark.serial +@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="Requires socket.AF_UNIX support") +def test_serve_unix_domain_socket(ds_unix_domain_socket_server): + _, uds = ds_unix_domain_socket_server + transport = httpx.HTTPTransport(uds=uds) + client = httpx.Client(transport=transport) + response = client.get("http://localhost/_memory.json") + assert { + "database": "_memory", + "path": "/_memory", + "tables": [], + }.items() <= response.json().items() From de2a1063284834ff86cb8d7c693717609d0d647e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 16:46:49 -0700 Subject: [PATCH 0390/1866] Ran Black, refs #1388 --- tests/test_cli_serve_server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index 73439125..1c31e2a3 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -25,7 +25,9 @@ def test_serve_localhost_https(ds_localhost_https_server): @pytest.mark.serial -@pytest.mark.skipif(not hasattr(socket, "AF_UNIX"), reason="Requires socket.AF_UNIX support") +@pytest.mark.skipif( + not hasattr(socket, "AF_UNIX"), reason="Requires socket.AF_UNIX support" +) def test_serve_unix_domain_socket(ds_unix_domain_socket_server): _, uds = ds_unix_domain_socket_server transport = httpx.HTTPTransport(uds=uds) From d792fc7cf5fde8fa748168e48c3183266a3a419f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Jul 2021 17:29:42 -0700 Subject: [PATCH 0391/1866] Simplified nginx config examples --- docs/deploying.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index f3680034..ce4acc9d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -144,14 +144,12 @@ Here is an example of an `nginx `__ configuration file that events { worker_connections 1024; } - http { server { listen 80; location /my-datasette { - proxy_pass http://127.0.0.1:8009/my-datasette; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://127.0.0.1:8009/my-datasette; + proxy_set_header Host $host; } } } @@ -166,9 +164,8 @@ You can also use the ``--uds`` option to Datasette to listen on a Unix domain so server { listen 80; location / { - proxy_pass http://datasette; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://datasette; + proxy_set_header Host $host; } } upstream datasette { From f83c84fd51d144036924ae77d99f12b0a69e7e6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jul 2021 18:36:18 -0700 Subject: [PATCH 0392/1866] Update asgiref requirement from <3.4.0,>=3.2.10 to >=3.2.10,<3.5.0 (#1386) Updates the requirements on [asgiref](https://github.com/django/asgiref) to permit the latest version. - [Release notes](https://github.com/django/asgiref/releases) - [Changelog](https://github.com/django/asgiref/blob/main/CHANGELOG.txt) - [Commits](https://github.com/django/asgiref/commits) --- updated-dependencies: - dependency-name: asgiref dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a651d32..2541be1f 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( include_package_data=True, python_requires=">=3.6", install_requires=[ - "asgiref>=3.2.10,<3.4.0", + "asgiref>=3.2.10,<3.5.0", "click>=7.1.1,<8.1.0", "click-default-group~=1.2.2", "Jinja2>=2.10.3,<3.1.0", From 4054e96a3914e821d0880a40a7284aaa9db1eaaa Mon Sep 17 00:00:00 2001 From: Aslak Raanes Date: Tue, 13 Jul 2021 19:42:27 +0200 Subject: [PATCH 0393/1866] Update deploying.rst (#1392) Use same base url for Apache as in the example --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index ce4acc9d..3be36df4 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -185,7 +185,7 @@ For `Apache `__, you can use the ``ProxyPass`` direct Then add these directives to proxy traffic:: - ProxyPass /datasette-prefix/ http://127.0.0.1:8009/datasette-prefix/ + ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPreserveHost On The `ProxyPreserveHost On `__ directive ensures that the original ``Host:`` header from the incoming request is passed through to Datasette. Datasette needs this to correctly assemble links to other pages using the :ref:`datasette_absolute_url` method. From d71cac498138ddd86f18607b9043e70286ea884a Mon Sep 17 00:00:00 2001 From: Aslak Raanes Date: Tue, 13 Jul 2021 20:32:49 +0200 Subject: [PATCH 0394/1866] How to configure Unix domain sockets with Apache Example on how to use Unix domain socket option on Apache. Not testet. (Usually I would have used [`ProxyPassReverse`](https://httpd.apache.org/docs/current/mod/mod_proxy.html#proxypassreverse) in combination with `ProxyPass` , i.e. ```apache ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPassReverse /my-datasette/ http://127.0.0.1:8009/my-datasette/ ``` and ```apache ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ ProxyPassReverse /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ ``` --- docs/deploying.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/deploying.rst b/docs/deploying.rst index 3be36df4..c471fad6 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -188,4 +188,8 @@ Then add these directives to proxy traffic:: ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPreserveHost On +Using ``--uds`` you can use Unix domain sockets similiar to the Nginx example: + + ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ + The `ProxyPreserveHost On `__ directive ensures that the original ``Host:`` header from the incoming request is passed through to Datasette. Datasette needs this to correctly assemble links to other pages using the :ref:`datasette_absolute_url` method. From 7f4c854db1ed8c15338e9cf42d2a3f0c92e3b7b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Jul 2021 11:45:32 -0700 Subject: [PATCH 0395/1866] rST fix --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index c471fad6..366c9d61 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -188,7 +188,7 @@ Then add these directives to proxy traffic:: ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPreserveHost On -Using ``--uds`` you can use Unix domain sockets similiar to the Nginx example: +Using ``--uds`` you can use Unix domain sockets similiar to the nginx example:: ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ From 2c4cd7141abb5115eff00ed7aef002af39d51989 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Jul 2021 16:15:48 -0700 Subject: [PATCH 0396/1866] Consistently use /my-datasette in examples --- docs/deploying.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index 366c9d61..c3e3e123 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -163,8 +163,8 @@ You can also use the ``--uds`` option to Datasette to listen on a Unix domain so http { server { listen 80; - location / { - proxy_pass http://datasette; + location /my-datasette { + proxy_pass http://datasette/my-datasette; proxy_set_header Host $host; } } @@ -173,7 +173,7 @@ You can also use the ``--uds`` option to Datasette to listen on a Unix domain so } } -Then run Datasette with ``datasette --uds /tmp/datasette.sock path/to/database.db``. +Then run Datasette with ``datasette --uds /tmp/datasette.sock path/to/database.db --setting base_url /my-datasette/``. Apache proxy configuration -------------------------- From ba11ef27edd6981eeb26d7ecf5aa236707f5f8ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Jul 2021 22:43:13 -0700 Subject: [PATCH 0397/1866] Clarify when to use systemd restart --- docs/deploying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index c3e3e123..31d123e9 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -65,7 +65,7 @@ You can start the Datasette process running using the following:: sudo systemctl daemon-reload sudo systemctl start datasette.service -You may need to restart the Datasette service after making changes to its ``metadata.json`` configuration or the ``datasette.service`` file. You can do that using:: +You will need to restart the Datasette service after making changes to its ``metadata.json`` configuration or adding a new database file to that directory. You can do that using:: sudo systemctl restart datasette.service From a6c8e7fa4cffdeff84e9e755dcff4788fd6154b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Jul 2021 17:05:18 -0700 Subject: [PATCH 0398/1866] Big performance boost for faceting, closes #1394 --- datasette/views/table.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1bda7496..876a0c81 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -674,12 +674,11 @@ class TableView(RowTableShared): else: page_size = self.ds.page_size - sql_no_limit = ( - "select {select_all_columns} from {table_name} {where}{order_by}".format( + sql_no_order_no_limit = ( + "select {select_all_columns} from {table_name} {where}".format( select_all_columns=select_all_columns, table_name=escape_sqlite(table), where=where_clause, - order_by=order_by, ) ) sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( @@ -736,7 +735,7 @@ class TableView(RowTableShared): self.ds, request, database, - sql=sql_no_limit, + sql=sql_no_order_no_limit, params=params, table=table, metadata=table_metadata, From 7ea678db228504004b8d32f813c838b1dcfd317a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Jul 2021 17:19:31 -0700 Subject: [PATCH 0399/1866] Warn about potential changes to get_metadata hook, refs #1384 --- docs/plugin_hooks.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b687a6e7..6c2ad1e5 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1150,6 +1150,9 @@ get_metadata(datasette, key, database, table) This hook is responsible for returning a dictionary corresponding to Datasette :ref:`metadata`. This function is passed the ``database``, ``table`` and ``key`` which were passed to the upstream internal request for metadata. Regardless, it is important to return a global metadata object, where ``"databases": []`` would be a top-level key. The dictionary returned here, will be merged with, and overwritten by, the contents of the physical ``metadata.yaml`` if one is present. +.. warning:: + The design of this plugin hook does not currently provide a mechanism for interacting with async code, and may change in the future. See `issue 1384 `__. + .. code-block:: python @hookimpl From e27dd7c12c2a6977560dbc0005e32c55d9d759f4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Jul 2021 17:32:33 -0700 Subject: [PATCH 0400/1866] Release 0.58 Refs #1365, #1371, #1377, #1384, #1387, #1388, #1389, #1394 --- datasette/version.py | 2 +- docs/changelog.rst | 19 +++++++++---------- docs/plugin_hooks.rst | 2 ++ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index e5a29931..0f94b605 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.58a1" +__version__ = "0.58" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index bcd8b987..201cf4b7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,22 +4,21 @@ Changelog ========= -.. _v0_58a1: +.. _v0_58: -0.58a1 (2021-06-24) -------------------- +0.58 (2021-07-14) +----------------- +- New ``datasette --uds /tmp/datasette.sock`` option for binding Datasette to a Unix domain socket, see :ref:`proxy documentation ` (:issue:`1388`) +- ``"searchmode": "raw"`` table metadata option for defaulting a table to executing SQLite full-text search syntax without first escaping it, see :ref:`full_text_search_advanced_queries`. (:issue:`1389`) +- New plugin hook: :ref:`plugin_hook_get_metadata`, for returning custom metadata for an instance, database or table. Thanks, Brandon Roberts! (:issue:`1384`) - New plugin hook: :ref:`plugin_hook_skip_csrf`, for opting out of CSRF protection based on the incoming request. (:issue:`1377`) +- The :ref:`menu_links() `, :ref:`table_actions() ` and :ref:`database_actions() ` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`) +- Major performance improvement for Datasette faceting. (:issue:`1394`) +- Improved documentation for :ref:`deploying_proxy` to recommend using ``ProxyPreservehost On`` with Apache. (:issue:`1387`) - ``POST`` requests to endpoints that do not support that HTTP verb now return a 405 error. - ``db.path`` can now be provided as a ``pathlib.Path`` object, useful when writing unit tests for plugins. Thanks, Chris Amico. (:issue:`1365`) -.. _v0_58a0: - -0.58a0 (2021-06-09) -------------------- - -- The :ref:`menu_links() `, :ref:`table_actions() ` and :ref:`database_actions() ` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`) - .. _v0_57_1: 0.57.1 (2021-06-08) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 6c2ad1e5..63258e2f 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1130,6 +1130,8 @@ This example will disable CSRF protection for that specific URL path: If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request. +.. _plugin_hook_get_metadata: + get_metadata(datasette, key, database, table) --------------------------------------------- From 084cfe1e00e1a4c0515390a513aca286eeea20c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Jul 2021 18:00:39 -0700 Subject: [PATCH 0401/1866] Removed out-of-date datasette serve help from README --- README.md | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/README.md b/README.md index 5682f59e..55160afe 100644 --- a/README.md +++ b/README.md @@ -53,39 +53,6 @@ Now visiting http://localhost:8001/History/downloads will show you a web interfa ![Downloads table rendered by datasette](https://static.simonwillison.net/static/2017/datasette-downloads.png) -## datasette serve options - - Usage: datasette serve [OPTIONS] [FILES]... - - Serve up specified SQLite database files with a web UI - - Options: - -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means - only connections from the local machine will be - allowed. Use 0.0.0.0 to listen to all IPs and - allow access from other machines. - -p, --port INTEGER Port for server, defaults to 8001 - --reload Automatically reload if code or metadata change - detected - useful for development - --cors Enable CORS by serving Access-Control-Allow- - Origin: * - --load-extension PATH Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette - inspect" - -m, --metadata FILENAME Path to JSON file containing license/source - metadata - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static STATIC MOUNT mountpoint:path-to-directory for serving static - files - --memory Make /_memory database available - --config CONFIG Set config option using configname:value - docs.datasette.io/en/stable/config.html - --version-note TEXT Additional note to show on /-/versions - --help-config Show available config options - --help Show this message and exit. - ## metadata.json If you want to include licensing and source information in the generated datasette website you can do so using a JSON file that looks something like this: From 721a8d3cd4937f888efd2b52d5a61f0e25b484e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Jul 2021 18:51:36 -0700 Subject: [PATCH 0402/1866] Hopeful fix for publish problem in #1396 --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 727f9933..54e582f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -115,5 +115,5 @@ jobs: -t $REPO:${GITHUB_REF#refs/tags/} \ --build-arg VERSION=${GITHUB_REF#refs/tags/} . docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest - docker push $REPO:${VERSION_TAG} + docker push $REPO:${GITHUB_REF#refs/tags/} docker push $REPO:latest From dd5ee8e66882c94343cd3f71920878c6cfd0da41 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 15 Jul 2021 23:26:06 -0700 Subject: [PATCH 0403/1866] Removed some unused imports I found these with: flake8 datasette | grep unus --- datasette/app.py | 1 - datasette/default_magic_parameters.py | 1 - datasette/facets.py | 2 -- datasette/utils/__init__.py | 4 +--- datasette/utils/asgi.py | 2 -- datasette/views/base.py | 1 - datasette/views/index.py | 2 +- setup.py | 2 -- 8 files changed, 2 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0b909968..5976d8b8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -58,7 +58,6 @@ from .utils import ( parse_metadata, resolve_env_secrets, to_css_class, - HASH_LENGTH, ) from .utils.asgi import ( AsgiLifespan, diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index 0f8f397e..19382207 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -1,5 +1,4 @@ from datasette import hookimpl -from datasette.utils import escape_fts import datetime import os import time diff --git a/datasette/facets.py b/datasette/facets.py index 250734fd..f74e2d01 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -1,6 +1,5 @@ import json import urllib -import re from datasette import hookimpl from datasette.database import QueryInterrupted from datasette.utils import ( @@ -8,7 +7,6 @@ from datasette.utils import ( path_with_added_args, path_with_removed_args, detect_json1, - InvalidSql, sqlite3, ) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1e193862..aec5a55b 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -5,7 +5,6 @@ from collections import OrderedDict, namedtuple, Counter import base64 import hashlib import inspect -import itertools import json import markupsafe import mergedeep @@ -17,10 +16,9 @@ import time import types import shutil import urllib -import numbers import yaml from .shutil_backport import copytree -from .sqlite import sqlite3, sqlite_version, supports_table_xinfo +from .sqlite import sqlite3, supports_table_xinfo # From https://www.sqlite.org/lang_keywords.html diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 63bf4926..5fa03b0a 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -3,9 +3,7 @@ from datasette.utils import MultiParams from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl from pathlib import Path -from html import escape from http.cookies import SimpleCookie, Morsel -import re import aiofiles import aiofiles.os diff --git a/datasette/views/base.py b/datasette/views/base.py index a87a0e77..cd584899 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -9,7 +9,6 @@ import urllib import pint from datasette import __version__ -from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, diff --git a/datasette/views/index.py b/datasette/views/index.py index 8ac117a6..e37643f9 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -2,7 +2,7 @@ import hashlib import json from datasette.utils import check_visibility, CustomJSONEncoder -from datasette.utils.asgi import Response, Forbidden +from datasette.utils.asgi import Response from datasette.version import __version__ from .base import BaseView diff --git a/setup.py b/setup.py index 2541be1f..cfc1e484 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ -from re import VERBOSE from setuptools import setup, find_packages import os -import sys def get_long_description(): From c00f29affcafce8314366852ba1a0f5a7dd25690 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Jul 2021 12:44:58 -0700 Subject: [PATCH 0404/1866] Fix for race condition in refresh_schemas(), closes #1231 --- datasette/app.py | 7 +++++++ datasette/utils/internal_db.py | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 5976d8b8..5f348cb5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -224,6 +224,7 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() + self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb if memory or crossdb or not self.files: self.add_database(Database(self, is_memory=True), name="_memory") @@ -332,6 +333,12 @@ class Datasette: self.client = DatasetteClient(self) async def refresh_schemas(self): + if self._refresh_schemas_lock.locked(): + return + async with self._refresh_schemas_lock: + await self._refresh_schemas() + + async def _refresh_schemas(self): internal_db = self.databases["_internal"] if not self.internal_db_created: await init_internal_db(internal_db) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e92625d5..40fe719e 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,7 +5,7 @@ async def init_internal_db(db): await db.execute_write( textwrap.dedent( """ - CREATE TABLE databases ( + CREATE TABLE IF NOT EXISTS databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, @@ -18,7 +18,7 @@ async def init_internal_db(db): await db.execute_write( textwrap.dedent( """ - CREATE TABLE tables ( + CREATE TABLE IF NOT EXISTS tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): await db.execute_write( textwrap.dedent( """ - CREATE TABLE columns ( + CREATE TABLE IF NOT EXISTS columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -54,7 +54,7 @@ async def init_internal_db(db): await db.execute_write( textwrap.dedent( """ - CREATE TABLE indexes ( + CREATE TABLE IF NOT EXISTS indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -73,7 +73,7 @@ async def init_internal_db(db): await db.execute_write( textwrap.dedent( """ - CREATE TABLE foreign_keys ( + CREATE TABLE IF NOT EXISTS foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, From c73af5dd72305f6a01ea94a2c76d52e5e26de38b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Jul 2021 12:46:13 -0700 Subject: [PATCH 0405/1866] Release 0.58.1 Refs #1231, #1396 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0f94b605..1b7b7350 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.58" +__version__ = "0.58.1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 201cf4b7..6a951935 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_58_1: + +0.58.1 (2021-07-16) +------------------- + +- Fix for an intermittent race condition caused by the ``refresh_schemas()`` internal function. (:issue:`1231`) + .. _v0_58: 0.58 (2021-07-14) From 6f1731f3055a5119cc393c118937d749405a1617 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Jul 2021 12:38:09 -0700 Subject: [PATCH 0406/1866] Updated cookiecutter installation link --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 6afee1c3..bd60a4b6 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -41,7 +41,7 @@ Plugins that can be installed should be written as Python packages using a ``set The quickest way to start writing one an installable plugin is to use the `datasette-plugin `__ cookiecutter template. This creates a new plugin structure for you complete with an example test and GitHub Actions workflows for testing and publishing your plugin. -`Install cookiecutter `__ and then run this command to start building a plugin using the template:: +`Install cookiecutter `__ and then run this command to start building a plugin using the template:: cookiecutter gh:simonw/datasette-plugin From eccfeb0871dd4bc27870faf64f80ac68e5b6bc0d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Jul 2021 16:16:46 -0700 Subject: [PATCH 0407/1866] register_routes() plugin hook datasette argument, closes #1404 --- datasette/app.py | 2 +- datasette/hookspecs.py | 2 +- docs/plugin_hooks.rst | 7 +++++-- tests/fixtures.py | 1 + tests/plugins/my_plugin_2.py | 10 ++++++++++ tests/test_plugins.py | 19 +++++++++++++++++++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 5f348cb5..2596ca50 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -960,7 +960,7 @@ class Datasette: """Returns an ASGI app function that serves the whole of Datasette""" routes = [] - for routes_to_add in pm.hook.register_routes(): + for routes_to_add in pm.hook.register_routes(datasette=self): for regex, view_fn in routes_to_add: routes.append((regex, wrap_view(view_fn, self))) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 07b2f5ba..3ef0d4f5 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -75,7 +75,7 @@ def register_facet_classes(): @hookspec -def register_routes(): +def register_routes(datasette): """Register URL routes: return a list of (regex, view_function) pairs""" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 63258e2f..4700763c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -529,8 +529,11 @@ Examples: `datasette-atom `_, `dataset .. _plugin_register_routes: -register_routes() ------------------ +register_routes(datasette) +-------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` Register additional view functions to execute for specified URL routes. diff --git a/tests/fixtures.py b/tests/fixtures.py index dce94876..93b7dce2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -70,6 +70,7 @@ EXPECTED_PLUGINS = [ "extra_template_vars", "menu_links", "permission_allowed", + "register_routes", "render_cell", "startup", "table_actions", diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index b70372f3..f7a3f1c0 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -1,4 +1,5 @@ from datasette import hookimpl +from datasette.utils.asgi import Response from functools import wraps import markupsafe import json @@ -167,3 +168,12 @@ def table_actions(datasette, database, table, actor, request): return [{"href": datasette.urls.instance(), "label": label}] return inner + + +@hookimpl +def register_routes(datasette): + config = datasette.plugin_config("register-route-demo") + if not config: + return + path = config["path"] + return [(r"/{}/$".format(path), lambda: Response.text(path.upper()))] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 7a626ce5..0c01b7ae 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -648,6 +648,25 @@ def test_hook_register_routes(app_client, path, body): assert body == response.text +@pytest.mark.parametrize("configured_path", ("path1", "path2")) +def test_hook_register_routes_with_datasette(configured_path): + with make_app_client( + metadata={ + "plugins": { + "register-route-demo": { + "path": configured_path, + } + } + } + ) as client: + response = client.get(f"/{configured_path}/") + assert response.status == 200 + assert configured_path.upper() == response.text + # Other one should 404 + other_path = [p for p in ("path1", "path2") if configured_path != p][0] + assert client.get(f"/{other_path}/").status == 404 + + def test_hook_register_routes_post(app_client): response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True) assert 200 == response.status From 121e10c29c5b412fddf0326939f1fe46c3ad9d4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jul 2021 16:30:12 -0700 Subject: [PATCH 0408/1866] Doumentation and test for utils.parse_metadata(), closes #1405 --- docs/internals.rst | 18 ++++++++++++++++++ tests/test_utils.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 98df998a..1e41cacd 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -795,3 +795,21 @@ By default all actors are denied access to the ``view-database`` permission for Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. + +.. _internals_utils: + +The datasette.utils module +========================== + +The ``datasette.utils`` module contains various utility functions used by Datasette. As a general rule you should consider anything in this module to be unstable - functions and classes here could change without warning or be removed entirely between Datasette releases, without being mentioned in the release notes. + +The exception to this rule is anythang that is documented here. If you find a need for an undocumented utility function in your own work, consider `opening an issue `__ requesting that the function you are using be upgraded to documented and supported status. + +.. _internals_utils_parse_metadata: + +parse_metadata(content) +----------------------- + +This function accepts a string containing either JSON or YAML, expected to be of the format described in :ref:`metadata`. It returns a nested Python dictionary representing the parsed data from that string. + +If the metadata cannot be parsed as either JSON or YAML the function will raise a ``utils.BadMetadataError`` exception. diff --git a/tests/test_utils.py b/tests/test_utils.py index be3daf2e..97b70ee5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -610,3 +610,19 @@ async def test_initial_path_for_datasette(tmp_path_factory, dbs, expected_path): ) path = await utils.initial_path_for_datasette(datasette) assert path == expected_path + + +@pytest.mark.parametrize( + "content,expected", + ( + ("title: Hello", {"title": "Hello"}), + ('{"title": "Hello"}', {"title": "Hello"}), + ("{{ this }} is {{ bad }}", None), + ), +) +def test_parse_metadata(content, expected): + if expected is None: + with pytest.raises(utils.BadMetadataError): + utils.parse_metadata(content) + else: + assert utils.parse_metadata(content) == expected From 2b1c535c128984cc0ee2a097ecaa3ab638ae2a5b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jul 2021 17:44:16 -0700 Subject: [PATCH 0409/1866] pytest.mark.serial for any test using isolated_filesystem(), refs #1406 --- tests/test_package.py | 3 ++- tests/test_publish_cloudrun.py | 7 +++++++ tests/test_publish_heroku.py | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_package.py b/tests/test_package.py index bb939643..76693d2f 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -2,7 +2,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock import pathlib -import json +import pytest class CaptureDockerfile: @@ -24,6 +24,7 @@ CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data """.strip() +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.cli.call") def test_package(mock_call, mock_which): diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 7881ebae..826860d7 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -6,6 +6,7 @@ import pytest import textwrap +@pytest.mark.serial @mock.patch("shutil.which") def test_publish_cloudrun_requires_gcloud(mock_which): mock_which.return_value = False @@ -27,6 +28,7 @@ def test_publish_cloudrun_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -75,6 +77,7 @@ Service name: input-service ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -103,6 +106,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -147,6 +151,7 @@ def test_publish_cloudrun_memory( ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -225,6 +230,7 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): } == json.loads(metadata) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -280,6 +286,7 @@ def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): assert expected == dockerfile +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index c011ab43..acbdafeb 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,8 +1,10 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import pytest +@pytest.mark.serial @mock.patch("shutil.which") def test_publish_heroku_requires_heroku(mock_which): mock_which.return_value = False @@ -15,6 +17,7 @@ def test_publish_heroku_requires_heroku(mock_which): assert "Publishing to Heroku requires heroku" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") @@ -44,6 +47,7 @@ def test_publish_heroku_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") @@ -79,6 +83,7 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") From 74b775e20f870de921ca3c09a75fe69e1c199fc7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jul 2021 17:50:45 -0700 Subject: [PATCH 0410/1866] Use consistent pattern for test before deploy, refs #1406 --- .github/workflows/deploy-latest.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index d9f23f7d..849adb40 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -29,7 +29,9 @@ jobs: python -m pip install -e .[docs] python -m pip install sphinx-to-sqlite==0.1a1 - name: Run tests - run: pytest + run: | + pytest -n auto -m "not serial" + pytest -m "serial" - name: Build fixtures.db run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db - name: Build docs.db From e55cd9dc3f2d920d5cf6d8581ce49937a6ccc44d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jul 2021 18:16:58 -0700 Subject: [PATCH 0411/1866] Try passing a directory to isolated_filesystem(), refs #1406 --- tests/test_package.py | 10 ++++----- tests/test_publish_cloudrun.py | 39 ++++++++++++++++------------------ tests/test_publish_heroku.py | 25 +++++++++++----------- 3 files changed, 34 insertions(+), 40 deletions(-) diff --git a/tests/test_package.py b/tests/test_package.py index 76693d2f..a72eef94 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -2,7 +2,6 @@ from click.testing import CliRunner from datasette import cli from unittest import mock import pathlib -import pytest class CaptureDockerfile: @@ -24,15 +23,14 @@ CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data """.strip() -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.cli.call") -def test_package(mock_call, mock_which): +def test_package(mock_call, mock_which, tmp_path_factory): mock_which.return_value = True runner = CliRunner() capture = CaptureDockerfile() mock_call.side_effect = capture - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"]) @@ -43,12 +41,12 @@ def test_package(mock_call, mock_which): @mock.patch("shutil.which") @mock.patch("datasette.cli.call") -def test_package_with_port(mock_call, mock_which): +def test_package_with_port(mock_call, mock_which, tmp_path_factory): mock_which.return_value = True capture = CaptureDockerfile() mock_call.side_effect = capture runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 826860d7..d91b7646 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -6,12 +6,11 @@ import pytest import textwrap -@pytest.mark.serial @mock.patch("shutil.which") -def test_publish_cloudrun_requires_gcloud(mock_which): +def test_publish_cloudrun_requires_gcloud(mock_which, tmp_path_factory): mock_which.return_value = False runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) @@ -28,13 +27,12 @@ def test_publish_cloudrun_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @mock.patch("datasette.publish.cloudrun.get_existing_services") def test_publish_cloudrun_prompts_for_service( - mock_get_existing_services, mock_call, mock_output, mock_which + mock_get_existing_services, mock_call, mock_output, mock_which, tmp_path_factory ): mock_get_existing_services.return_value = [ {"name": "existing", "created": "2019-01-01", "url": "http://www.example.com/"} @@ -42,7 +40,7 @@ def test_publish_cloudrun_prompts_for_service( mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( @@ -77,15 +75,14 @@ Service name: input-service ) -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") -def test_publish_cloudrun(mock_call, mock_output, mock_which): +def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( @@ -106,7 +103,6 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): ) -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -121,12 +117,12 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which): ], ) def test_publish_cloudrun_memory( - mock_call, mock_output, mock_which, memory, should_fail + mock_call, mock_output, mock_which, memory, should_fail, tmp_path_factory ): mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( @@ -151,16 +147,17 @@ def test_publish_cloudrun_memory( ) -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") -def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): +def test_publish_cloudrun_plugin_secrets( + mock_call, mock_output, mock_which, tmp_path_factory +): mock_which.return_value = True mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") with open("metadata.yml", "w") as fp: @@ -230,16 +227,17 @@ def test_publish_cloudrun_plugin_secrets(mock_call, mock_output, mock_which): } == json.loads(metadata) -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") -def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): +def test_publish_cloudrun_apt_get_install( + mock_call, mock_output, mock_which, tmp_path_factory +): mock_which.return_value = True mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( @@ -286,7 +284,6 @@ def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): assert expected == dockerfile -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -302,13 +299,13 @@ def test_publish_cloudrun_apt_get_install(mock_call, mock_output, mock_which): ], ) def test_publish_cloudrun_extra_options( - mock_call, mock_output, mock_which, extra_options, expected + mock_call, mock_output, mock_which, extra_options, expected, tmp_path_factory ): mock_which.return_value = True mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index acbdafeb..a591bcf8 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,15 +1,13 @@ from click.testing import CliRunner from datasette import cli from unittest import mock -import pytest -@pytest.mark.serial @mock.patch("shutil.which") -def test_publish_heroku_requires_heroku(mock_which): +def test_publish_heroku_requires_heroku(mock_which, tmp_path_factory): mock_which.return_value = False runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) @@ -17,15 +15,16 @@ def test_publish_heroku_requires_heroku(mock_which): assert "Publishing to Heroku requires heroku" in result.output -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") -def test_publish_heroku_installs_plugin(mock_call, mock_check_output, mock_which): +def test_publish_heroku_installs_plugin( + mock_call, mock_check_output, mock_which, tmp_path_factory +): mock_which.return_value = True mock_check_output.side_effect = lambda s: {"['heroku', 'plugins']": b""}[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("t.db", "w") as fp: fp.write("data") result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") @@ -47,11 +46,10 @@ def test_publish_heroku_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") -def test_publish_heroku(mock_call, mock_check_output, mock_which): +def test_publish_heroku(mock_call, mock_check_output, mock_which, tmp_path_factory): mock_which.return_value = True mock_check_output.side_effect = lambda s: { "['heroku', 'plugins']": b"heroku-builds", @@ -59,7 +57,7 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', }[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( @@ -83,11 +81,12 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which): ) -@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") -def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which): +def test_publish_heroku_plugin_secrets( + mock_call, mock_check_output, mock_which, tmp_path_factory +): mock_which.return_value = True mock_check_output.side_effect = lambda s: { "['heroku', 'plugins']": b"heroku-builds", @@ -95,7 +94,7 @@ def test_publish_heroku_plugin_secrets(mock_call, mock_check_output, mock_which) "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', }[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(): + with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): with open("test.db", "w") as fp: fp.write("data") result = runner.invoke( From b46856391de5a819a85d1dd970428cbc702be94a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jul 2021 17:44:16 -0700 Subject: [PATCH 0412/1866] pytest.mark.serial for any test using isolated_filesystem(), refs #1406 --- tests/test_package.py | 2 ++ tests/test_publish_cloudrun.py | 7 +++++++ tests/test_publish_heroku.py | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/tests/test_package.py b/tests/test_package.py index a72eef94..98e701bf 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -2,6 +2,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock import pathlib +import pytest class CaptureDockerfile: @@ -23,6 +24,7 @@ CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data """.strip() +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.cli.call") def test_package(mock_call, mock_which, tmp_path_factory): diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index d91b7646..ee0c9c95 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -6,6 +6,7 @@ import pytest import textwrap +@pytest.mark.serial @mock.patch("shutil.which") def test_publish_cloudrun_requires_gcloud(mock_which, tmp_path_factory): mock_which.return_value = False @@ -27,6 +28,7 @@ def test_publish_cloudrun_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -75,6 +77,7 @@ Service name: input-service ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -103,6 +106,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -147,6 +151,7 @@ def test_publish_cloudrun_memory( ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -227,6 +232,7 @@ def test_publish_cloudrun_plugin_secrets( } == json.loads(metadata) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @@ -284,6 +290,7 @@ def test_publish_cloudrun_apt_get_install( assert expected == dockerfile +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index a591bcf8..1fe02e08 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,8 +1,10 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import pytest +@pytest.mark.serial @mock.patch("shutil.which") def test_publish_heroku_requires_heroku(mock_which, tmp_path_factory): mock_which.return_value = False @@ -15,6 +17,7 @@ def test_publish_heroku_requires_heroku(mock_which, tmp_path_factory): assert "Publishing to Heroku requires heroku" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") @@ -46,6 +49,7 @@ def test_publish_heroku_invalid_database(mock_which): assert "Path 'woop.db' does not exist" in result.output +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") @@ -81,6 +85,7 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which, tmp_path_facto ) +@pytest.mark.serial @mock.patch("shutil.which") @mock.patch("datasette.publish.heroku.check_output") @mock.patch("datasette.publish.heroku.call") From 96b1d0b7b42928e657b1aebcc95d55e4685690e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Jul 2021 11:48:33 -0700 Subject: [PATCH 0413/1866] Attempted fix for too-long UDS bug in #1407 --- tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 34a64efc..215853b3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -157,8 +157,11 @@ def ds_localhost_https_server(tmp_path_factory): @pytest.fixture(scope="session") def ds_unix_domain_socket_server(tmp_path_factory): - socket_folder = tmp_path_factory.mktemp("uds") - uds = str(socket_folder / "datasette.sock") + # This used to use tmp_path_factory.mktemp("uds") but that turned out to + # produce paths that were too long to use as UDS on macOS, see + # https://github.com/simonw/datasette/issues/1407 - so I switched to + # using tempfile.gettempdir() + uds = str(pathlib.Path(tempfile.gettempdir()) / "datasette.sock") ds_proc = subprocess.Popen( ["datasette", "--memory", "--uds", uds], stdout=subprocess.PIPE, From ff253f5242e4b0b5d85d29d38b8461feb5ea997a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Jul 2021 11:49:08 -0700 Subject: [PATCH 0414/1866] Replace all uses of runner.isolated_filesystem, refs #1406 --- tests/test_package.py | 27 ++- tests/test_publish_cloudrun.py | 422 ++++++++++++++++----------------- tests/test_publish_heroku.py | 127 +++++----- 3 files changed, 284 insertions(+), 292 deletions(-) diff --git a/tests/test_package.py b/tests/test_package.py index 98e701bf..02ed1775 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import os import pathlib import pytest @@ -32,12 +33,12 @@ def test_package(mock_call, mock_which, tmp_path_factory): runner = CliRunner() capture = CaptureDockerfile() mock_call.side_effect = capture - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"]) - assert 0 == result.exit_code - mock_call.assert_has_calls([mock.call(["docker", "build", "."])]) + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke(cli.cli, ["package", "test.db", "--secret", "sekrit"]) + assert 0 == result.exit_code + mock_call.assert_has_calls([mock.call(["docker", "build", "."])]) assert EXPECTED_DOCKERFILE.format(port=8001) == capture.captured @@ -48,11 +49,11 @@ def test_package_with_port(mock_call, mock_which, tmp_path_factory): capture = CaptureDockerfile() mock_call.side_effect = capture runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, ["package", "test.db", "-p", "8080", "--secret", "sekrit"] - ) - assert 0 == result.exit_code + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, ["package", "test.db", "-p", "8080", "--secret", "sekrit"] + ) + assert 0 == result.exit_code assert EXPECTED_DOCKERFILE.format(port=8080) == capture.captured diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index ee0c9c95..47f59d72 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -2,6 +2,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock import json +import os import pytest import textwrap @@ -11,12 +12,12 @@ import textwrap def test_publish_cloudrun_requires_gcloud(mock_which, tmp_path_factory): mock_which.return_value = False runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) - assert result.exit_code == 1 - assert "Publishing to Google Cloud requires gcloud" in result.output + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke(cli.cli, ["publish", "cloudrun", "test.db"]) + assert result.exit_code == 1 + assert "Publishing to Google Cloud requires gcloud" in result.output @mock.patch("shutil.which") @@ -42,39 +43,32 @@ def test_publish_cloudrun_prompts_for_service( mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" - ) - assert ( - """ -Please provide a service name for this deployment - -Using an existing service name will over-write it - -Your existing services: - - existing - created 2019-01-01 - http://www.example.com/ - -Service name: input-service -""".strip() - == result.output.strip() - ) - assert 0 == result.exit_code - tag = "gcr.io/myproject/datasette" - mock_call.assert_has_calls( - [ - mock.call(f"gcloud builds submit --tag {tag}", shell=True), - mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( - tag - ), - shell=True, + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db"], input="input-service" + ) + assert ( + "Please provide a service name for this deployment\n\n" + "Using an existing service name will over-write it\n\n" + "Your existing services:\n\n" + " existing - created 2019-01-01 - http://www.example.com/\n\n" + "Service name: input-service" + ) == result.output.strip() + assert 0 == result.exit_code + tag = "gcr.io/myproject/datasette" + mock_call.assert_has_calls( + [ + mock.call(f"gcloud builds submit --tag {tag}", shell=True), + mock.call( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service".format( + tag ), - ] - ) + shell=True, + ), + ] + ) @pytest.mark.serial @@ -85,25 +79,25 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] - ) - assert 0 == result.exit_code - tag = f"gcr.io/{mock_output.return_value}/datasette" - mock_call.assert_has_calls( - [ - mock.call(f"gcloud builds submit --tag {tag}", shell=True), - mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test".format( - tag - ), - shell=True, + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, ["publish", "cloudrun", "test.db", "--service", "test"] + ) + assert 0 == result.exit_code + tag = f"gcr.io/{mock_output.return_value}/datasette" + mock_call.assert_has_calls( + [ + mock.call(f"gcloud builds submit --tag {tag}", shell=True), + mock.call( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test".format( + tag ), - ] - ) + shell=True, + ), + ] + ) @pytest.mark.serial @@ -126,29 +120,29 @@ def test_publish_cloudrun_memory( mock_output.return_value = "myproject" mock_which.return_value = True runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, - ["publish", "cloudrun", "test.db", "--service", "test", "--memory", memory], - ) - if should_fail: - assert 2 == result.exit_code - return - assert 0 == result.exit_code - tag = f"gcr.io/{mock_output.return_value}/datasette" - mock_call.assert_has_calls( - [ - mock.call(f"gcloud builds submit --tag {tag}", shell=True), - mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --memory {}".format( - tag, memory - ), - shell=True, + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, + ["publish", "cloudrun", "test.db", "--service", "test", "--memory", memory], + ) + if should_fail: + assert 2 == result.exit_code + return + assert 0 == result.exit_code + tag = f"gcr.io/{mock_output.return_value}/datasette" + mock_call.assert_has_calls( + [ + mock.call(f"gcloud builds submit --tag {tag}", shell=True), + mock.call( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --memory {}".format( + tag, memory ), - ] - ) + shell=True, + ), + ] + ) @pytest.mark.serial @@ -162,74 +156,74 @@ def test_publish_cloudrun_plugin_secrets( mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - with open("metadata.yml", "w") as fp: - fp.write( - textwrap.dedent( - """ - title: Hello from metadata YAML - plugins: - datasette-auth-github: - foo: bar + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + with open("metadata.yml", "w") as fp: + fp.write( + textwrap.dedent( """ - ).strip() - ) - result = runner.invoke( - cli.cli, - [ - "publish", - "cloudrun", - "test.db", - "--metadata", - "metadata.yml", - "--service", - "datasette", - "--plugin-secret", - "datasette-auth-github", - "client_id", - "x-client-id", - "--show-files", - "--secret", - "x-secret", - ], + title: Hello from metadata YAML + plugins: + datasette-auth-github: + foo: bar + """ + ).strip() ) - assert result.exit_code == 0 - dockerfile = ( - result.output.split("==== Dockerfile ====\n")[1] - .split("\n====================\n")[0] - .strip() - ) - expected = textwrap.dedent( - r""" - FROM python:3.8 - COPY . /app - WORKDIR /app + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--metadata", + "metadata.yml", + "--service", + "datasette", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + "--show-files", + "--secret", + "x-secret", + ], + ) + assert result.exit_code == 0 + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = textwrap.dedent( + r""" + FROM python:3.8 + COPY . /app + WORKDIR /app - ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' - ENV DATASETTE_SECRET 'x-secret' - RUN pip install -U datasette - RUN datasette inspect test.db --inspect-file inspect-data.json - ENV PORT 8001 - EXPOSE 8001 - CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --setting force_https_urls on --port $PORT""" - ).strip() - assert expected == dockerfile - metadata = ( - result.output.split("=== metadata.json ===\n")[1] - .split("\n==== Dockerfile ====\n")[0] - .strip() - ) - assert { - "title": "Hello from metadata YAML", - "plugins": { - "datasette-auth-github": { - "foo": "bar", - "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"}, - } + ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id' + ENV DATASETTE_SECRET 'x-secret' + RUN pip install -U datasette + RUN datasette inspect test.db --inspect-file inspect-data.json + ENV PORT 8001 + EXPOSE 8001 + CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --setting force_https_urls on --port $PORT""" + ).strip() + assert expected == dockerfile + metadata = ( + result.output.split("=== metadata.json ===\n")[1] + .split("\n==== Dockerfile ====\n")[0] + .strip() + ) + assert { + "title": "Hello from metadata YAML", + "plugins": { + "datasette-auth-github": { + "client_id": {"$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID"}, + "foo": "bar", }, - } == json.loads(metadata) + }, + } == json.loads(metadata) @pytest.mark.serial @@ -243,51 +237,51 @@ def test_publish_cloudrun_apt_get_install( mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, - [ - "publish", - "cloudrun", - "test.db", - "--service", - "datasette", - "--show-files", - "--secret", - "x-secret", - "--apt-get-install", - "ripgrep", - "--spatialite", - ], - ) - assert result.exit_code == 0 - dockerfile = ( - result.output.split("==== Dockerfile ====\n")[1] - .split("\n====================\n")[0] - .strip() - ) - expected = textwrap.dedent( - r""" - FROM python:3.8 - COPY . /app - WORKDIR /app + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--service", + "datasette", + "--show-files", + "--secret", + "x-secret", + "--apt-get-install", + "ripgrep", + "--spatialite", + ], + ) + assert result.exit_code == 0 + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + expected = textwrap.dedent( + r""" + FROM python:3.8 + COPY . /app + WORKDIR /app - RUN apt-get update && \ - apt-get install -y ripgrep python3-dev gcc libsqlite3-mod-spatialite && \ - rm -rf /var/lib/apt/lists/* + RUN apt-get update && \ + apt-get install -y ripgrep python3-dev gcc libsqlite3-mod-spatialite && \ + rm -rf /var/lib/apt/lists/* - ENV DATASETTE_SECRET 'x-secret' - ENV SQLITE_EXTENSIONS '/usr/lib/x86_64-linux-gnu/mod_spatialite.so' - RUN pip install -U datasette - RUN datasette inspect test.db --inspect-file inspect-data.json - ENV PORT 8001 - EXPOSE 8001 - CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --setting force_https_urls on --port $PORT - """ - ).strip() - assert expected == dockerfile + ENV DATASETTE_SECRET 'x-secret' + ENV SQLITE_EXTENSIONS '/usr/lib/x86_64-linux-gnu/mod_spatialite.so' + RUN pip install -U datasette + RUN datasette inspect test.db --inspect-file inspect-data.json + ENV PORT 8001 + EXPOSE 8001 + CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --setting force_https_urls on --port $PORT + """ + ).strip() + assert expected == dockerfile @pytest.mark.serial @@ -312,32 +306,32 @@ def test_publish_cloudrun_extra_options( mock_output.return_value = "myproject" runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, - [ - "publish", - "cloudrun", - "test.db", - "--service", - "datasette", - "--show-files", - "--extra-options", - extra_options, - ], - ) - assert result.exit_code == 0 - dockerfile = ( - result.output.split("==== Dockerfile ====\n")[1] - .split("\n====================\n")[0] - .strip() - ) - last_line = dockerfile.split("\n")[-1] - extra_options = ( - last_line.split("--inspect-file inspect-data.json")[1] - .split("--port")[0] - .strip() - ) - assert extra_options == expected + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "cloudrun", + "test.db", + "--service", + "datasette", + "--show-files", + "--extra-options", + extra_options, + ], + ) + assert result.exit_code == 0 + dockerfile = ( + result.output.split("==== Dockerfile ====\n")[1] + .split("\n====================\n")[0] + .strip() + ) + last_line = dockerfile.split("\n")[-1] + extra_options = ( + last_line.split("--inspect-file inspect-data.json")[1] + .split("--port")[0] + .strip() + ) + assert extra_options == expected diff --git a/tests/test_publish_heroku.py b/tests/test_publish_heroku.py index 1fe02e08..b5a8af73 100644 --- a/tests/test_publish_heroku.py +++ b/tests/test_publish_heroku.py @@ -1,6 +1,7 @@ from click.testing import CliRunner from datasette import cli from unittest import mock +import os import pytest @@ -9,12 +10,12 @@ import pytest def test_publish_heroku_requires_heroku(mock_which, tmp_path_factory): mock_which.return_value = False runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) - assert result.exit_code == 1 - assert "Publishing to Heroku requires heroku" in result.output + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke(cli.cli, ["publish", "heroku", "test.db"]) + assert result.exit_code == 1 + assert "Publishing to Heroku requires heroku" in result.output @pytest.mark.serial @@ -27,11 +28,11 @@ def test_publish_heroku_installs_plugin( mock_which.return_value = True mock_check_output.side_effect = lambda s: {"['heroku', 'plugins']": b""}[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("t.db", "w") as fp: - fp.write("data") - result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") - assert 0 != result.exit_code + os.chdir(tmp_path_factory.mktemp("runner")) + with open("t.db", "w") as fp: + fp.write("data") + result = runner.invoke(cli.cli, ["publish", "heroku", "t.db"], input="y\n") + assert 0 != result.exit_code mock_check_output.assert_has_calls( [mock.call(["heroku", "plugins"]), mock.call(["heroku", "apps:list", "--json"])] ) @@ -61,28 +62,26 @@ def test_publish_heroku(mock_call, mock_check_output, mock_which, tmp_path_facto "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', }[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, ["publish", "heroku", "test.db", "--tar", "gtar"] - ) - assert 0 == result.exit_code, result.output - mock_call.assert_has_calls( - [ - mock.call( - [ - "heroku", - "builds:create", - "-a", - "f", - "--include-vcs-ignore", - "--tar", - "gtar", - ] - ), - ] - ) + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke(cli.cli, ["publish", "heroku", "test.db", "--tar", "gtar"]) + assert 0 == result.exit_code, result.output + mock_call.assert_has_calls( + [ + mock.call( + [ + "heroku", + "builds:create", + "-a", + "f", + "--include-vcs-ignore", + "--tar", + "gtar", + ] + ), + ] + ) @pytest.mark.serial @@ -99,35 +98,33 @@ def test_publish_heroku_plugin_secrets( "['heroku', 'apps:create', 'datasette', '--json']": b'{"name": "f"}', }[repr(s)] runner = CliRunner() - with runner.isolated_filesystem(tmp_path_factory.mktemp("runner")): - with open("test.db", "w") as fp: - fp.write("data") - result = runner.invoke( - cli.cli, - [ - "publish", - "heroku", - "test.db", - "--plugin-secret", - "datasette-auth-github", - "client_id", - "x-client-id", - ], - ) - assert 0 == result.exit_code, result.output - mock_call.assert_has_calls( - [ - mock.call( - [ - "heroku", - "config:set", - "-a", - "f", - "DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id", - ] - ), - mock.call( - ["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"] - ), - ] - ) + os.chdir(tmp_path_factory.mktemp("runner")) + with open("test.db", "w") as fp: + fp.write("data") + result = runner.invoke( + cli.cli, + [ + "publish", + "heroku", + "test.db", + "--plugin-secret", + "datasette-auth-github", + "client_id", + "x-client-id", + ], + ) + assert 0 == result.exit_code, result.output + mock_call.assert_has_calls( + [ + mock.call( + [ + "heroku", + "config:set", + "-a", + "f", + "DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id", + ] + ), + mock.call(["heroku", "builds:create", "-a", "f", "--include-vcs-ignore"]), + ] + ) From 4adca0d85077fe504e98cd7487343e76ccf25be5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 31 Jul 2021 17:58:11 -0700 Subject: [PATCH 0415/1866] No hidden SQL on canned query pages, closes #1411 --- datasette/templates/query.html | 2 +- tests/test_html.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b6c74883..543561d8 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -44,7 +44,7 @@ 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} - + {% if not canned_query %}{% endif %} {% endif %} {% if named_parameter_values %} diff --git a/tests/test_html.py b/tests/test_html.py index aee6bce1..9f5b99e3 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1238,6 +1238,17 @@ def test_show_hide_sql_query(app_client): ] == [(hidden["name"], hidden["value"]) for hidden in hiddens] +def test_canned_query_with_hide_has_no_hidden_sql(app_client): + # For a canned query the show/hide should NOT have a hidden SQL field + # https://github.com/simonw/datasette/issues/1411 + response = app_client.get("/fixtures/neighborhood_search?_hide_sql=1") + soup = Soup(response.body, "html.parser") + hiddens = soup.find("form").select("input[type=hidden]") + assert [ + ("_hide_sql", "1"), + ] == [(hidden["name"], hidden["value"]) for hidden in hiddens] + + def test_extra_where_clauses(app_client): response = app_client.get( "/fixtures/facetable?_where=neighborhood='Dogpatch'&_where=city_id=1" From a679d0de87031e3de9013fc299ba2cbd75808684 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 09:11:18 -0700 Subject: [PATCH 0416/1866] Fixed spelling of 'receive' in a bunch of places --- docs/internals.rst | 2 +- docs/plugin_hooks.rst | 4 ++-- tests/plugins/my_plugin.py | 4 ++-- tests/plugins/my_plugin_2.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 1e41cacd..cfc4f6d5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -149,7 +149,7 @@ Create a ``Response`` object and then use ``await response.asgi_send(send)``, pa .. code-block:: python - async def require_authorization(scope, recieve, send): + async def require_authorization(scope, receive, send): response = Response.text( "401 Authorization Required", headers={ diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4700763c..269cb1c9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -678,7 +678,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att def asgi_wrapper(datasette): def wrap_with_databases_header(app): @wraps(app) - async def add_x_databases_header(scope, recieve, send): + async def add_x_databases_header(scope, receive, send): async def wrapped_send(event): if event["type"] == "http.response.start": original_headers = event.get("headers") or [] @@ -691,7 +691,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att ], } await send(event) - await app(scope, recieve, wrapped_send) + await app(scope, receive, wrapped_send) return add_x_databases_header return wrap_with_databases_header diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 0e625623..59ac8add 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -178,11 +178,11 @@ def actor_from_request(datasette, request): @hookimpl def asgi_wrapper(): def wrap(app): - async def maybe_set_actor_in_scope(scope, recieve, send): + async def maybe_set_actor_in_scope(scope, receive, send): if b"_actor_in_scope" in scope.get("query_string", b""): scope = dict(scope, actor={"id": "from-scope"}) print(scope) - await app(scope, recieve, send) + await app(scope, receive, send) return maybe_set_actor_in_scope diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index f7a3f1c0..ba298fd4 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -77,7 +77,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette def asgi_wrapper(datasette): def wrap_with_databases_header(app): @wraps(app) - async def add_x_databases_header(scope, recieve, send): + async def add_x_databases_header(scope, receive, send): async def wrapped_send(event): if event["type"] == "http.response.start": original_headers = event.get("headers") or [] @@ -94,7 +94,7 @@ def asgi_wrapper(datasette): } await send(event) - await app(scope, recieve, wrapped_send) + await app(scope, receive, wrapped_send) return add_x_databases_header From 54b6e96ee8aa553b6671e341a1944f93f3fb89c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 09:12:48 -0700 Subject: [PATCH 0417/1866] Use optional rich dependency to render tracebacks, closes #1416 --- datasette/app.py | 8 ++++++++ datasette/cli.py | 8 ++++++++ setup.py | 1 + 3 files changed, 17 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 2596ca50..edd5ab87 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -81,6 +81,11 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ +try: + import rich +except ImportError: + rich = None + app_root = Path(__file__).parent.parent # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 @@ -1270,6 +1275,9 @@ class DatasetteRouter: pdb.post_mortem(exception.__traceback__) + if rich is not None: + rich.console.Console().print_exception(show_locals=True) + title = None if isinstance(exception, Forbidden): status = 403 diff --git a/datasette/cli.py b/datasette/cli.py index 09aebcc8..e53f3d8e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -31,6 +31,14 @@ from .utils.sqlite import sqlite3 from .utils.testing import TestClient from .version import __version__ +# Use Rich for tracebacks if it is installed +try: + from rich.traceback import install + + install(show_locals=True) +except ImportError: + pass + class Config(click.ParamType): # This will be removed in Datasette 1.0 in favour of class Setting diff --git a/setup.py b/setup.py index cfc1e484..c69b9b00 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ setup( "pytest-timeout>=1.4.2,<1.5", "trustme>=0.7,<0.9", ], + "rich": ["rich"], }, tests_require=["datasette[test]"], classifiers=[ From 2208c3c68e552d343e6a2872ff6e559fca9d1b38 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 09:36:38 -0700 Subject: [PATCH 0418/1866] Spelling corrections plus CI job for codespell * Use codespell to check spelling in documentation, refs #1417 * Fixed spelling errors spotted by codespell, closes #1417 * Make codespell a docs dependency See also this TIL: https://til.simonwillison.net/python/codespell --- .github/workflows/spellcheck.yml | 25 +++++++++++++++++++++++++ docs/authentication.rst | 4 ++-- docs/changelog.rst | 8 ++++---- docs/codespell-ignore-words.txt | 1 + docs/deploying.rst | 2 +- docs/internals.rst | 6 +++--- docs/performance.rst | 2 +- docs/plugin_hooks.rst | 2 +- docs/publish.rst | 2 +- docs/settings.rst | 2 +- docs/sql_queries.rst | 2 +- setup.py | 2 +- 12 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/spellcheck.yml create mode 100644 docs/codespell-ignore-words.txt diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml new file mode 100644 index 00000000..d498e173 --- /dev/null +++ b/.github/workflows/spellcheck.yml @@ -0,0 +1,25 @@ +name: Check spelling in documentation + +on: [push, pull_request] + +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + name: Configure pip caching + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + pip install -e '.[docs]' + - name: Check spelling + run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/docs/authentication.rst b/docs/authentication.rst index 62ed7e8b..0d98cf82 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -60,7 +60,7 @@ The key question the permissions system answers is this: **Actors** are :ref:`described above `. -An **action** is a string describing the action the actor would like to perfom. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``. +An **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``. A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource. @@ -73,7 +73,7 @@ Permissions with potentially harmful effects should default to *deny*. Plugin au Defining permissions with "allow" blocks ---------------------------------------- -The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perfom a permission. +The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perform a permission. The most basic form of allow block is this (`allow demo `__, `deny demo `__): diff --git a/docs/changelog.rst b/docs/changelog.rst index 6a951935..883cb3eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -426,7 +426,7 @@ See also `Datasette 0.49: The annotated release notes `__ for conversations about the project that go beyond just bug reports and issues. - Datasette can now be installed on macOS using Homebrew! Run ``brew install simonw/datasette/datasette``. See :ref:`installation_homebrew`. (:issue:`335`) - Two new commands: ``datasette install name-of-plugin`` and ``datasette uninstall name-of-plugin``. These are equivalent to ``pip install`` and ``pip uninstall`` but automatically run in the same virtual environment as Datasette, so users don't have to figure out where that virtual environment is - useful for installations created using Homebrew or ``pipx``. See :ref:`plugins_installing`. (:issue:`925`) -- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the repsonse. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`) +- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`) .. _v0_46: @@ -500,7 +500,7 @@ New plugin hooks Smaller changes ~~~~~~~~~~~~~~~ -- Cascading view permissons - so if a user has ``view-table`` they can view the table page even if they do not have ``view-database`` or ``view-instance``. (:issue:`832`) +- Cascading view permissions - so if a user has ``view-table`` they can view the table page even if they do not have ``view-database`` or ``view-instance``. (:issue:`832`) - CSRF protection no longer applies to ``Authentication: Bearer token`` requests or requests without cookies. (:issue:`835`) - ``datasette.add_message()`` now works inside plugins. (:issue:`864`) - Workaround for "Too many open files" error in test runs. (:issue:`846`) @@ -714,7 +714,7 @@ Also in this release: * Datasette now has a *pattern portfolio* at ``/-/patterns`` - e.g. https://latest.datasette.io/-/patterns. This is a page that shows every Datasette user interface component in one place, to aid core development and people building custom CSS themes. (:issue:`151`) * SQLite `PRAGMA functions `__ such as ``pragma_table_info(tablename)`` are now allowed in Datasette SQL queries. (:issue:`761`) * Datasette pages now consistently return a ``content-type`` of ``text/html; charset=utf-8"``. (:issue:`752`) -* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibilty with the `Mangum `__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (`#719 `__) +* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibility with the `Mangum `__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (`#719 `__) * Installation documentation now covers how to :ref:`installation_pipx`. (:issue:`756`) * Improved the documentation for :ref:`full_text_search`. (:issue:`748`) @@ -1169,7 +1169,7 @@ Documentation improvements plus a fix for publishing to Zeit Now. New plugin hooks, improved database view support and an easier way to use more recent versions of SQLite. - New ``publish_subcommand`` plugin hook. A plugin can now add additional ``datasette publish`` publishers in addition to the default ``now`` and ``heroku``, both of which have been refactored into default plugins. :ref:`publish_subcommand documentation `. Closes :issue:`349` -- New ``render_cell`` plugin hook. Plugins can now customize how values are displayed in the HTML tables produced by Datasette's browseable interface. `datasette-json-html `__ and `datasette-render-images `__ are two new plugins that use this hook. :ref:`render_cell documentation `. Closes :issue:`352` +- New ``render_cell`` plugin hook. Plugins can now customize how values are displayed in the HTML tables produced by Datasette's browsable interface. `datasette-json-html `__ and `datasette-render-images `__ are two new plugins that use this hook. :ref:`render_cell documentation `. Closes :issue:`352` - New ``extra_body_script`` plugin hook, enabling plugins to provide additional JavaScript that should be added to the page footer. :ref:`extra_body_script documentation `. - ``extra_css_urls`` and ``extra_js_urls`` hooks now take additional optional parameters, allowing them to be more selective about which pages they apply to. :ref:`Documentation `. - You can now use the :ref:`sortable_columns metadata setting ` to explicitly enable sort-by-column in the interface for database views, as well as for specific tables. diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt new file mode 100644 index 00000000..a625cde5 --- /dev/null +++ b/docs/codespell-ignore-words.txt @@ -0,0 +1 @@ +AddWordsToIgnoreHere diff --git a/docs/deploying.rst b/docs/deploying.rst index 31d123e9..83d9e4dd 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -188,7 +188,7 @@ Then add these directives to proxy traffic:: ProxyPass /my-datasette/ http://127.0.0.1:8009/my-datasette/ ProxyPreserveHost On -Using ``--uds`` you can use Unix domain sockets similiar to the nginx example:: +Using ``--uds`` you can use Unix domain sockets similar to the nginx example:: ProxyPass /my-datasette/ unix:/tmp/datasette.sock|http://localhost/my-datasette/ diff --git a/docs/internals.rst b/docs/internals.rst index cfc4f6d5..058a8969 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -431,13 +431,13 @@ It offers the following methods: ``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response Execute an internal GET request against that path. -``await datasette.client.post(path, **kwargs)`` - returns HTTPX Respons +``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters. ``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response Execute an internal OPTIONS request. -``await datasette.client.head(path, **kwargs)`` - returns HTTPX Respons +``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response Execute an internal HEAD request. ``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response @@ -714,7 +714,7 @@ The ``Database`` class also provides properties and methods for introspecting th List of names of tables in the database. ``await db.view_names()`` - list of strings - List of names of views in tha database. + List of names of views in the database. ``await db.table_columns(table)`` - list of strings Names of columns in a specific table. diff --git a/docs/performance.rst b/docs/performance.rst index b9e38e2f..bcf3208e 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -39,7 +39,7 @@ Then later you can start Datasette against the ``counts.json`` file and use it t datasette -i data.db --inspect-file=counts.json -You need to use the ``-i`` immutable mode against the databse file here or the counts from the JSON file will be ignored. +You need to use the ``-i`` immutable mode against the database file here or the counts from the JSON file will be ignored. You will rarely need to use this optimization in every-day use, but several of the ``datasette publish`` commands described in :ref:`publishing` use this optimization for better performance when deploying a database file to a hosting provider. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 269cb1c9..10ec2cf1 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -664,7 +664,7 @@ Return an `ASGI `__ middleware wrapper function th This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code. -You can write your ASGI code directly against the low-level specification, or you can use the middleware utilites provided by an ASGI framework such as `Starlette `__. +You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette `__. This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases: diff --git a/docs/publish.rst b/docs/publish.rst index cbd18a00..f6895f53 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -165,7 +165,7 @@ You can now run the resulting container like so:: This exposes port 8001 inside the container as port 8081 on your host machine, so you can access the application at ``http://localhost:8081/`` -You can customize the port that is exposed by the countainer using the ``--port`` option:: +You can customize the port that is exposed by the container using the ``--port`` option:: datasette package mydatabase.db --port 8080 diff --git a/docs/settings.rst b/docs/settings.rst index c246d33a..7cc4bae0 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -349,7 +349,7 @@ Using secrets with datasette publish The :ref:`cli_publish` and :ref:`cli_package` commands both generate a secret for you automatically when Datasette is deployed. -This means that every time you deploy a new version of a Datasette project, a new secret will be generated. This will cause signed cookies to become inalid on every fresh deploy. +This means that every time you deploy a new version of a Datasette project, a new secret will be generated. This will cause signed cookies to become invalid on every fresh deploy. You can fix this by creating a secret that will be used for multiple deploys and passing it using the ``--secret`` option:: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index e9077f70..3049593d 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -403,7 +403,7 @@ Datasette can execute joins across multiple databases if it is started with the If it is started in this way, the ``/_memory`` page can be used to execute queries that join across multiple databases. -References to tables in attached databases should be preceeded by the database name and a period. +References to tables in attached databases should be preceded by the database name and a period. For example, this query will show a list of tables across both of the above databases: diff --git a/setup.py b/setup.py index c69b9b00..65e99848 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( """, setup_requires=["pytest-runner"], extras_require={ - "docs": ["sphinx_rtd_theme", "sphinx-autobuild"], + "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"], "test": [ "pytest>=5.2.2,<6.3.0", "pytest-xdist>=2.2.1,<2.4", From cd8b7bee8fb5c1cdce7c8dbfeb0166011abc72c6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 10:03:08 -0700 Subject: [PATCH 0419/1866] Run codespell against datasette source code too, refs #1417 --- .github/workflows/spellcheck.yml | 4 +++- datasette/hookspecs.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d498e173..2e24d3eb 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -22,4 +22,6 @@ jobs: run: | pip install -e '.[docs]' - name: Check spelling - run: codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + run: | + codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 3ef0d4f5..f31ce538 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -86,12 +86,12 @@ def actor_from_request(datasette, request): @hookspec def permission_allowed(datasette, actor, action, resource): - """Check if actor is allowed to perfom this action - return True, False or None""" + """Check if actor is allowed to perform this action - return True, False or None""" @hookspec def canned_queries(datasette, database, actor): - """Return a dictonary of canned query definitions or an awaitable function that returns them""" + """Return a dictionary of canned query definitions or an awaitable function that returns them""" @hookspec From a1f383035698da8bf188659390af6e53ffeec940 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Aug 2021 22:20:50 -0700 Subject: [PATCH 0420/1866] --cpu option for datasette publish cloudrun, closes #1420 --- datasette/publish/cloudrun.py | 13 +++++- docs/datasette-publish-cloudrun-help.txt | 1 + tests/test_docs.py | 2 +- tests/test_publish_cloudrun.py | 51 ++++++++++++++++-------- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index bad223a1..1fabcafd 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -36,6 +36,11 @@ def publish_subcommand(publish): callback=_validate_memory, help="Memory to allocate in Cloud Run, e.g. 1Gi", ) + @click.option( + "--cpu", + type=click.Choice(["1", "2", "4"]), + help="Number of vCPUs to allocate in Cloud Run", + ) @click.option( "--apt-get-install", "apt_get_extras", @@ -66,6 +71,7 @@ def publish_subcommand(publish): spatialite, show_files, memory, + cpu, apt_get_extras, ): fail_if_publish_binary_not_installed( @@ -151,8 +157,11 @@ def publish_subcommand(publish): image_id = f"gcr.io/{project}/{name}" check_call(f"gcloud builds submit --tag {image_id}", shell=True) check_call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( - image_id, service, " --memory {}".format(memory) if memory else "" + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format( + image_id, + service, + " --memory {}".format(memory) if memory else "", + " --cpu {}".format(cpu) if cpu else "", ), shell=True, ) diff --git a/docs/datasette-publish-cloudrun-help.txt b/docs/datasette-publish-cloudrun-help.txt index 3d05efb6..34481b40 100644 --- a/docs/datasette-publish-cloudrun-help.txt +++ b/docs/datasette-publish-cloudrun-help.txt @@ -28,5 +28,6 @@ Options: --spatialite Enable SpatialLite extension --show-files Output the generated Dockerfile and metadata.json --memory TEXT Memory to allocate in Cloud Run, e.g. 1Gi + --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run --apt-get-install TEXT Additional packages to apt-get install --help Show this message and exit. diff --git a/tests/test_docs.py b/tests/test_docs.py index efd267b9..d0cb036d 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -50,7 +50,7 @@ def test_help_includes(name, filename): # actual has "Usage: cli package [OPTIONS] FILES" # because it doesn't know that cli will be aliased to datasette expected = expected.replace("Usage: datasette", "Usage: cli") - assert expected == actual + assert expected == actual, "Run python update-docs-help.py to fix this" @pytest.fixture(scope="session") diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 47f59d72..9c8c38cf 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -105,17 +105,28 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @pytest.mark.parametrize( - "memory,should_fail", + "memory,cpu,expected_gcloud_args", [ - ["1Gi", False], - ["2G", False], - ["256Mi", False], - ["4", True], - ["GB", True], + ["1Gi", None, "--memory 1Gi"], + ["2G", None, "--memory 2G"], + ["256Mi", None, "--memory 256Mi"], + ["4", None, None], + ["GB", None, None], + [None, 1, "--cpu 1"], + [None, 2, "--cpu 2"], + [None, 3, None], + [None, 4, "--cpu 4"], + ["2G", 4, "--memory 2G --cpu 4"], ], ) -def test_publish_cloudrun_memory( - mock_call, mock_output, mock_which, memory, should_fail, tmp_path_factory +def test_publish_cloudrun_memory_cpu( + mock_call, + mock_output, + mock_which, + memory, + cpu, + expected_gcloud_args, + tmp_path_factory, ): mock_output.return_value = "myproject" mock_which.return_value = True @@ -123,22 +134,30 @@ def test_publish_cloudrun_memory( os.chdir(tmp_path_factory.mktemp("runner")) with open("test.db", "w") as fp: fp.write("data") - result = runner.invoke( - cli.cli, - ["publish", "cloudrun", "test.db", "--service", "test", "--memory", memory], - ) - if should_fail: + args = ["publish", "cloudrun", "test.db", "--service", "test"] + if memory: + args.extend(["--memory", memory]) + if cpu: + args.extend(["--cpu", str(cpu)]) + result = runner.invoke(cli.cli, args) + if expected_gcloud_args is None: assert 2 == result.exit_code return assert 0 == result.exit_code tag = f"gcr.io/{mock_output.return_value}/datasette" + expected_call = ( + "gcloud run deploy --allow-unauthenticated --platform=managed" + " --image {} test".format(tag) + ) + if memory: + expected_call += " --memory {}".format(memory) + if cpu: + expected_call += " --cpu {}".format(cpu) mock_call.assert_has_calls( [ mock.call(f"gcloud builds submit --tag {tag}", shell=True), mock.call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --memory {}".format( - tag, memory - ), + expected_call, shell=True, ), ] From acc22436622ff8476c30acf45ed60f54b4aaa5d9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 5 Aug 2021 08:47:18 -0700 Subject: [PATCH 0421/1866] Quotes around '.[test]' for zsh --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c3d0989a..8a638e0b 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -39,7 +39,7 @@ The next step is to create a virtual environment for your project and use it to # Now activate the virtual environment, so pip can install into it source venv/bin/activate # Install Datasette and its testing dependencies - python3 -m pip install -e .[test] + python3 -m pip install -e '.[test]' That last line does most of the work: ``pip install -e`` means "install this package in a way that allows me to edit the source code in place". The ``.[test]`` option means "use the setup.py in this directory and install the optional testing dependencies as well". From b7037f5ecea40dc5343250d08d741504b6dcb28f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 4 Aug 2021 19:58:09 -0700 Subject: [PATCH 0422/1866] Bit of breathing space on https://latest.datasette.io/fixtures/pragma_cache_size --- datasette/static/app.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index ad517c98..c6be1e97 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -497,6 +497,9 @@ label.sort_by_desc { width: auto; padding-right: 1em; } +pre#sql-query { + margin-bottom: 1em; +} form input[type=text], form input[type=search] { border: 1px solid #ccc; From 66e143c76e90f643dc11b6ced5433130c90a2455 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Aug 2021 22:09:00 -0700 Subject: [PATCH 0423/1866] New hide_sql canned query option, refs #1422 --- datasette/templates/query.html | 14 +++++++--- datasette/views/database.py | 32 +++++++++++++++++++-- docs/changelog.rst | 2 +- docs/sql_queries.rst | 25 +++++++++++++---- tests/fixtures.py | 1 + tests/test_html.py | 51 +++++++++++++++++++++++++++++++++- 6 files changed, 111 insertions(+), 14 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 543561d8..75f7f1b1 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -33,7 +33,9 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} {% if hide_sql %}(show){% else %}(hide){% endif %}{% endif %}

    +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} + ({{ show_hide_text }}) + {% endif %}

    {% if error %}

    {{ error }}

    {% endif %} @@ -44,8 +46,11 @@ 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} - {% if not canned_query %}{% endif %} - + {% if not canned_query %} + + {% endif %} {% endif %} {% if named_parameter_values %}

    Query parameters

    @@ -54,9 +59,10 @@ {% endfor %} {% endif %}

    - + {% if not hide_sql %}{% endif %} {% if canned_write %}{% endif %} + {{ show_hide_hidden }} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

    diff --git a/datasette/views/database.py b/datasette/views/database.py index 53bdceed..d9fe2b49 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -5,6 +5,8 @@ import json from markupsafe import Markup, escape from urllib.parse import parse_qsl, urlencode +import markupsafe + from datasette.utils import ( await_me_maybe, check_visibility, @@ -415,6 +417,29 @@ class QueryView(DataView): } ) ) + + show_hide_hidden = "" + if metadata.get("hide_sql"): + if bool(params.get("_show_sql")): + show_hide_link = path_with_removed_args(request, {"_show_sql"}) + show_hide_text = "hide" + show_hide_hidden = ( + '' + ) + else: + show_hide_link = path_with_added_args(request, {"_show_sql": 1}) + show_hide_text = "show" + else: + if bool(params.get("_hide_sql")): + show_hide_link = path_with_removed_args(request, {"_hide_sql"}) + show_hide_text = "show" + show_hide_hidden = ( + '' + ) + else: + show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) + show_hide_text = "hide" + hide_sql = show_hide_text == "show" return { "display_rows": display_rows, "custom_sql": True, @@ -425,9 +450,10 @@ class QueryView(DataView): "metadata": metadata, "config": self.ds.config_dict(), "request": request, - "path_with_added_args": path_with_added_args, - "path_with_removed_args": path_with_removed_args, - "hide_sql": "_hide_sql" in params, + "show_hide_link": show_hide_link, + "show_hide_text": show_hide_text, + "show_hide_hidden": markupsafe.Markup(show_hide_hidden), + "hide_sql": hide_sql, } return ( diff --git a/docs/changelog.rst b/docs/changelog.rst index 883cb3eb..d0fee19b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -674,7 +674,7 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`) * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`) * New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`metadata_page_size`. (:issue:`751`) -* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_default_fragment`. (:issue:`706`) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_options`. (:issue:`706`) * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`) * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`) diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 3049593d..407e4ba2 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -187,14 +187,28 @@ You can alternatively provide an explicit list of named parameters using the ``" order by neighborhood title: Search neighborhoods -.. _canned_queries_default_fragment: +.. _canned_queries_options: -Setting a default fragment -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Additional canned query options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Additional options can be specified for canned queries in the YAML or JSON configuration. + +hide_sql +++++++++ + +Canned queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. + +Add the ``"hide_sql": true`` option to hide the SQL query by default. + +fragment +++++++++ Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. -You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key: +You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key. + +This example demonstrates both ``fragment`` and ``hide_sql``: .. code-block:: json @@ -204,7 +218,8 @@ You can set a default fragment hash that will be included in the link to the can "queries": { "neighborhood_search": { "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", - "fragment": "fragment-goes-here" + "fragment": "fragment-goes-here", + "hide_sql": true } } } diff --git a/tests/fixtures.py b/tests/fixtures.py index 93b7dce2..873f9d55 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -360,6 +360,7 @@ METADATA = { "title": "Search neighborhoods", "description_html": "Demonstrating simple like search", "fragment": "fragment-goes-here", + "hide_sql": True, }, }, } diff --git a/tests/test_html.py b/tests/test_html.py index 9f5b99e3..b1b6c1f3 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1241,7 +1241,7 @@ def test_show_hide_sql_query(app_client): def test_canned_query_with_hide_has_no_hidden_sql(app_client): # For a canned query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 - response = app_client.get("/fixtures/neighborhood_search?_hide_sql=1") + response = app_client.get("/fixtures/pragma_cache_size?_hide_sql=1") soup = Soup(response.body, "html.parser") hiddens = soup.find("form").select("input[type=hidden]") assert [ @@ -1249,6 +1249,55 @@ def test_canned_query_with_hide_has_no_hidden_sql(app_client): ] == [(hidden["name"], hidden["value"]) for hidden in hiddens] +@pytest.mark.parametrize( + "hide_sql,querystring,expected_hidden,expected_show_hide_link,expected_show_hide_text", + ( + (False, "", None, "/_memory/one?_hide_sql=1", "hide"), + (False, "?_hide_sql=1", "_hide_sql", "/_memory/one", "show"), + (True, "", None, "/_memory/one?_show_sql=1", "show"), + (True, "?_show_sql=1", "_show_sql", "/_memory/one", "hide"), + ), +) +def test_canned_query_show_hide_metadata_option( + hide_sql, + querystring, + expected_hidden, + expected_show_hide_link, + expected_show_hide_text, +): + with make_app_client( + metadata={ + "databases": { + "_memory": { + "queries": { + "one": { + "sql": "select 1 + 1", + "hide_sql": hide_sql, + } + } + } + } + }, + memory=True, + ) as client: + expected_show_hide_fragment = '({})'.format( + expected_show_hide_link, expected_show_hide_text + ) + response = client.get("/_memory/one" + querystring) + html = response.text + show_hide_fragment = html.split('')[1].split( + "" + )[0] + assert show_hide_fragment == expected_show_hide_fragment + if expected_hidden: + assert ( + ''.format(expected_hidden) + in html + ) + else: + assert ' Date: Fri, 6 Aug 2021 22:14:44 -0700 Subject: [PATCH 0424/1866] Fix for rich.console sometimes not being available, refs #1416 --- datasette/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index edd5ab87..f2f75884 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1276,7 +1276,7 @@ class DatasetteRouter: pdb.post_mortem(exception.__traceback__) if rich is not None: - rich.console.Console().print_exception(show_locals=True) + rich.get_console().print_exception(show_locals=True) title = None if isinstance(exception, Forbidden): From 6dd14a1221d0324f9e3d6cfa10d2281d1eba4806 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Aug 2021 22:38:47 -0700 Subject: [PATCH 0425/1866] Improved links to example plugins --- docs/plugin_hooks.rst | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 10ec2cf1..200e0305 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -53,7 +53,7 @@ arguments and can be called like this:: select random_integer(1, 10); -Examples: `datasette-jellyfish `__, `datasette-jq `__, `datasette-haversine `__, `datasette-rure `__ +Examples: `datasette-jellyfish `__, `datasette-jq `__, `datasette-haversine `__, `datasette-rure `__ .. _plugin_hook_prepare_jinja2_environment: @@ -161,7 +161,7 @@ You can then use the new function in a template like so:: SQLite version: {{ sql_first("select sqlite_version()") }} -Examples: `datasette-search-all `_, `datasette-template-sql `_ +Examples: `datasette-search-all `_, `datasette-template-sql `_ .. _plugin_hook_extra_css_urls: @@ -210,7 +210,7 @@ This function can also return an awaitable function, useful if it needs to run a return inner -Examples: `datasette-cluster-map `_, `datasette-vega `_ +Examples: `datasette-cluster-map `_, `datasette-vega `_ .. _plugin_hook_extra_js_urls: @@ -257,7 +257,7 @@ If your code uses `JavaScript modules `_, `datasette-vega `_ +Examples: `datasette-cluster-map `_, `datasette-vega `_ .. _plugin_hook_extra_body_script: @@ -291,7 +291,7 @@ This will add the following to the end of your page: -Example: `datasette-cluster-map `_ +Example: `datasette-cluster-map `_ .. _plugin_hook_publish_subcommand: @@ -348,7 +348,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ ): # Your implementation goes here -Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ +Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ .. _plugin_hook_render_cell: @@ -420,7 +420,7 @@ If the value matches that pattern, the plugin returns an HTML link element: label=markupsafe.escape(data["label"] or "") or " " )) -Examples: `datasette-render-binary `_, `datasette-render-markdown `__, `datasette-json-html `__ +Examples: `datasette-render-binary `_, `datasette-render-markdown `__, `datasette-json-html `__ .. _plugin_register_output_renderer: @@ -525,7 +525,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th def can_render_demo(columns): return {"atom_id", "atom_title", "atom_updated"}.issubset(columns) -Examples: `datasette-atom `_, `datasette-ics `_ +Examples: `datasette-atom `_, `datasette-ics `_ .. _plugin_register_routes: @@ -583,7 +583,7 @@ The function can either return a :ref:`internals_response` or it can return noth See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes used by your plugin. -Examples: `datasette-auth-github `__, `datasette-psutil `__ +Examples: `datasette-auth-github `__, `datasette-psutil `__ .. _plugin_register_facet_classes: @@ -695,7 +695,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return add_x_databases_header return wrap_with_databases_header -Example: `datasette-cors `_ +Example: `datasette-cors `_ .. _plugin_hook_startup: @@ -743,7 +743,7 @@ Potential use-cases: await ds.invoke_startup() # Rest of test goes here -Examples: `datasette-saved-queries `__, `datasette-init `__ +Examples: `datasette-saved-queries `__, `datasette-init `__ .. _plugin_hook_canned_queries: @@ -812,7 +812,7 @@ The actor parameter can be used to include the currently authenticated actor in } for result in results} return inner -Example: `datasette-saved-queries `__ +Example: `datasette-saved-queries `__ .. _plugin_hook_actor_from_request: @@ -873,7 +873,7 @@ Instead of returning a dictionary, this function can return an awaitable functio return inner -Example: `datasette-auth-tokens `_ +Example: `datasette-auth-tokens `_ .. _plugin_hook_permission_allowed: @@ -932,7 +932,7 @@ Here's an example that allows users to view the ``admin_log`` table only if thei See :ref:`built-in permissions ` for a full list of permissions that are included in Datasette core. -Example: `datasette-permissions-sql `_ +Example: `datasette-permissions-sql `_ .. _plugin_hook_register_magic_parameters: @@ -1051,6 +1051,8 @@ This example adds a new menu item but only if the signed in user is ``"root"``: Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. +Examples: `datasette-search-all `_, `datasette-graphql `_ + .. _plugin_hook_table_actions: table_actions(datasette, actor, database, table, request) @@ -1089,6 +1091,8 @@ This example adds a new table action if the signed in user is ``"root"``: "label": "Edit schema for this table", }] +Example: `datasette-graphql `_ + .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) @@ -1108,6 +1112,8 @@ database_actions(datasette, actor, database, request) This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. +Example: `datasette-graphql `_ + .. _plugin_hook_skip_csrf: skip_csrf(datasette, scope) @@ -1172,3 +1178,5 @@ This hook is responsible for returning a dictionary corresponding to Datasette : # whatever we return here will be merged with any other plugins using this hook and # will be overwritten by a local metadata.yaml if one exists! return metadata + +Example: `datasette-remote-metadata plugin `__ From 61505dd0c6717cecdb73897e8613de9e9b7b6c42 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 6 Aug 2021 22:40:07 -0700 Subject: [PATCH 0426/1866] Release 0.59a0 Refs #1404, #1405, #1416, #1420, #1422 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 1b7b7350..05704728 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.58.1" +__version__ = "0.59a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d0fee19b..2cffef0f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v0_59a0: + +0.59a0 (2021-08-06) +------------------- + +- :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`) +- New ``hide_sql`` canned query option for defaulting to hiding the SQL quey used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`) +- New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`) +- If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`) +- ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`) + .. _v0_58_1: 0.58.1 (2021-07-16) From de5ce2e56339ad8966f417a4758f7c210c017dec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 10:37:51 -0700 Subject: [PATCH 0427/1866] datasette-pyinstrument --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 200e0305..64c56309 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -695,7 +695,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return add_x_databases_header return wrap_with_databases_header -Example: `datasette-cors `_ +Examples: `datasette-cors `__, `datasette-pyinstrument `__ .. _plugin_hook_startup: From 3bb6409a6cb8eaee32eb572423d9c0485a1dd917 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 16:04:42 -0700 Subject: [PATCH 0428/1866] render_cell() can now return an awaitable, refs --- datasette/views/database.py | 1 + datasette/views/table.py | 1 + docs/plugin_hooks.rst | 4 +++- tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 38 ++++++++++++++++++++++--------------- tests/test_api.py | 37 +++++++++++++++++++++++++++++++----- tests/test_plugins.py | 5 +++++ 7 files changed, 66 insertions(+), 21 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index d9fe2b49..f835dfac 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -361,6 +361,7 @@ class QueryView(DataView): database=database, datasette=self.ds, ) + plugin_value = await await_me_maybe(plugin_value) if plugin_value is not None: display_value = plugin_value else: diff --git a/datasette/views/table.py b/datasette/views/table.py index 876a0c81..3d25a1a5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -198,6 +198,7 @@ class RowTableShared(DataView): database=database, datasette=self.ds, ) + plugin_display_value = await await_me_maybe(plugin_display_value) if plugin_display_value is not None: display_value = plugin_display_value elif isinstance(value, bytes): diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 64c56309..5cdb1623 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -370,7 +370,7 @@ Lets you customize the display of values within table cells in the HTML table vi The name of the database ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value. @@ -378,6 +378,8 @@ If the hook returns a string, that string will be rendered in the table cell. If you want to return HTML markup you can do so by returning a ``jinja2.Markup`` object. +You can also return an awaitable function which returns a value. + Datasette will loop through all available ``render_cell`` hooks and display the value returned by the first one that does not return ``None``. Here is an example of a custom ``render_cell()`` plugin which looks for values that are a JSON string matching the following format:: diff --git a/tests/fixtures.py b/tests/fixtures.py index 873f9d55..880e4347 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -644,6 +644,7 @@ INSERT INTO simple_primary_key VALUES (1, 'hello'); INSERT INTO simple_primary_key VALUES (2, 'world'); INSERT INTO simple_primary_key VALUES (3, ''); INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); +INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 59ac8add..75c76ea8 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -97,21 +97,29 @@ def extra_body_script( @hookimpl def render_cell(value, column, table, database, datasette): - # Render some debug output in cell with value RENDER_CELL_DEMO - if value != "RENDER_CELL_DEMO": - return None - return json.dumps( - { - "column": column, - "table": table, - "database": database, - "config": datasette.plugin_config( - "name-of-plugin", - database=database, - table=table, - ), - } - ) + async def inner(): + # Render some debug output in cell with value RENDER_CELL_DEMO + if value == "RENDER_CELL_DEMO": + return json.dumps( + { + "column": column, + "table": table, + "database": database, + "config": datasette.plugin_config( + "name-of-plugin", + database=database, + table=table, + ), + } + ) + elif value == "RENDER_CELL_ASYNC": + return ( + await datasette.get_database(database).execute( + "select 'RENDER_CELL_ASYNC_RESULT'" + ) + ).single_value() + + return inner @hookimpl diff --git a/tests/test_api.py b/tests/test_api.py index 0049d76d..83cca521 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -415,7 +415,7 @@ def test_database_page(app_client): "name": "simple_primary_key", "columns": ["id", "content"], "primary_keys": ["id"], - "count": 4, + "count": 5, "hidden": False, "fts_table": None, "foreign_keys": { @@ -652,6 +652,7 @@ def test_custom_sql(app_client): {"content": "world"}, {"content": ""}, {"content": "RENDER_CELL_DEMO"}, + {"content": "RENDER_CELL_ASYNC"}, ] == data["rows"] assert ["content"] == data["columns"] assert "fixtures" == data["database"] @@ -693,6 +694,7 @@ def test_table_json(app_client): {"id": "2", "content": "world"}, {"id": "3", "content": ""}, {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, ] @@ -723,6 +725,7 @@ def test_table_shape_arrays(app_client): ["2", "world"], ["3", ""], ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], ] == response.json["rows"] @@ -736,7 +739,13 @@ def test_table_shape_arrayfirst(app_client): } ) ) - assert ["hello", "world", "", "RENDER_CELL_DEMO"] == response.json + assert [ + "hello", + "world", + "", + "RENDER_CELL_DEMO", + "RENDER_CELL_ASYNC", + ] == response.json def test_table_shape_objects(app_client): @@ -746,6 +755,7 @@ def test_table_shape_objects(app_client): {"id": "2", "content": "world"}, {"id": "3", "content": ""}, {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, ] == response.json["rows"] @@ -756,6 +766,7 @@ def test_table_shape_array(app_client): {"id": "2", "content": "world"}, {"id": "3", "content": ""}, {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, ] == response.json @@ -768,6 +779,7 @@ def test_table_shape_array_nl(app_client): {"id": "2", "content": "world"}, {"id": "3", "content": ""}, {"id": "4", "content": "RENDER_CELL_DEMO"}, + {"id": "5", "content": "RENDER_CELL_ASYNC"}, ] == results @@ -788,6 +800,7 @@ def test_table_shape_object(app_client): "2": {"id": "2", "content": "world"}, "3": {"id": "3", "content": ""}, "4": {"id": "4", "content": "RENDER_CELL_DEMO"}, + "5": {"id": "5", "content": "RENDER_CELL_ASYNC"}, } == response.json @@ -1145,12 +1158,21 @@ def test_searchable_invalid_column(app_client): ("/fixtures/simple_primary_key.json?content=hello", [["1", "hello"]]), ( "/fixtures/simple_primary_key.json?content__contains=o", - [["1", "hello"], ["2", "world"], ["4", "RENDER_CELL_DEMO"]], + [ + ["1", "hello"], + ["2", "world"], + ["4", "RENDER_CELL_DEMO"], + ], ), ("/fixtures/simple_primary_key.json?content__exact=", [["3", ""]]), ( "/fixtures/simple_primary_key.json?content__not=world", - [["1", "hello"], ["3", ""], ["4", "RENDER_CELL_DEMO"]], + [ + ["1", "hello"], + ["3", ""], + ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], + ], ), ], ) @@ -1163,7 +1185,11 @@ def test_table_filter_queries_multiple_of_same_type(app_client): response = app_client.get( "/fixtures/simple_primary_key.json?content__not=world&content__not=hello" ) - assert [["3", ""], ["4", "RENDER_CELL_DEMO"]] == response.json["rows"] + assert [ + ["3", ""], + ["4", "RENDER_CELL_DEMO"], + ["5", "RENDER_CELL_ASYNC"], + ] == response.json["rows"] @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") @@ -1293,6 +1319,7 @@ def test_view(app_client): {"upper_content": "WORLD", "content": "world"}, {"upper_content": "", "content": ""}, {"upper_content": "RENDER_CELL_DEMO", "content": "RENDER_CELL_DEMO"}, + {"upper_content": "RENDER_CELL_ASYNC", "content": "RENDER_CELL_ASYNC"}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 0c01b7ae..9bda7420 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -185,6 +185,11 @@ def test_hook_render_cell_demo(app_client): } == json.loads(td.string) +def test_hook_render_cell_async(app_client): + response = app_client.get("/fixtures?sql=select+'RENDER_CELL_ASYNC'") + assert b"RENDER_CELL_ASYNC_RESULT" in response.body + + def test_plugin_config(app_client): assert {"depth": "table"} == app_client.ds.plugin_config( "name-of-plugin", database="fixtures", table="sortable" From 818b0b76a2d58f7c2d850570efcdc22d345b4059 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 16:07:52 -0700 Subject: [PATCH 0429/1866] Test table render_cell async as well as query results, refs #1425 --- tests/test_plugins.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9bda7420..ec8ff0c5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -185,8 +185,11 @@ def test_hook_render_cell_demo(app_client): } == json.loads(td.string) -def test_hook_render_cell_async(app_client): - response = app_client.get("/fixtures?sql=select+'RENDER_CELL_ASYNC'") +@pytest.mark.parametrize( + "path", ("/fixtures?sql=select+'RENDER_CELL_ASYNC'", "/fixtures/simple_primary_key") +) +def test_hook_render_cell_async(app_client, path): + response = app_client.get(path) assert b"RENDER_CELL_ASYNC_RESULT" in response.body From f3c9edb376a13c09b5ecf97c7390f4e49efaadf2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 16:11:40 -0700 Subject: [PATCH 0430/1866] Fixed some tests I broke in #1425 --- tests/test_csv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index 3debf320..5e9406e7 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -11,6 +11,7 @@ EXPECTED_TABLE_CSV = """id,content 2,world 3, 4,RENDER_CELL_DEMO +5,RENDER_CELL_ASYNC """.replace( "\n", "\r\n" ) @@ -167,7 +168,7 @@ def test_csv_trace(app_client_with_trace): soup = Soup(response.text, "html.parser") assert ( soup.find("textarea").text - == "id,content\r\n1,hello\r\n2,world\r\n3,\r\n4,RENDER_CELL_DEMO\r\n" + == "id,content\r\n1,hello\r\n2,world\r\n3,\r\n4,RENDER_CELL_DEMO\r\n5,RENDER_CELL_ASYNC\r\n" ) assert "select id, content from simple_primary_key" in soup.find("pre").text From a390bdf9cef01d8723d025fc3348e81345ff4856 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 17:38:42 -0700 Subject: [PATCH 0431/1866] Stop using firstresult=True on render_cell, refs #1425 See https://github.com/simonw/datasette/issues/1425#issuecomment-894883664 --- datasette/hookspecs.py | 2 +- datasette/views/database.py | 14 +++++++++----- datasette/views/table.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index f31ce538..56c79d23 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -59,7 +59,7 @@ def publish_subcommand(publish): """Subcommands for 'datasette publish'""" -@hookspec(firstresult=True) +@hookspec def render_cell(value, column, table, database, datasette): """Customize rendering of HTML table cell values""" diff --git a/datasette/views/database.py b/datasette/views/database.py index f835dfac..29600659 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -354,16 +354,20 @@ class QueryView(DataView): display_value = value # Let the plugins have a go # pylint: disable=no-member - plugin_value = pm.hook.render_cell( + plugin_display_value = None + for candidate in pm.hook.render_cell( value=value, column=column, table=None, database=database, datasette=self.ds, - ) - plugin_value = await await_me_maybe(plugin_value) - if plugin_value is not None: - display_value = plugin_value + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value is not None: + display_value = plugin_display_value else: if value in ("", None): display_value = Markup(" ") diff --git a/datasette/views/table.py b/datasette/views/table.py index 3d25a1a5..456d8069 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -191,15 +191,19 @@ class RowTableShared(DataView): # First let the plugins have a go # pylint: disable=no-member - plugin_display_value = pm.hook.render_cell( + plugin_display_value = None + for candidate in pm.hook.render_cell( value=value, column=column, table=table, database=database, datasette=self.ds, - ) - plugin_display_value = await await_me_maybe(plugin_display_value) - if plugin_display_value is not None: + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: display_value = plugin_display_value elif isinstance(value, bytes): display_value = markupsafe.Markup( From ad90a72afa21b737b162e2bbdddc301a97d575cd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 18:13:03 -0700 Subject: [PATCH 0432/1866] Release 0.59a1 Refs #1425 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 05704728..f5fbfb3f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.59a0" +__version__ = "0.59a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2cffef0f..1406a7ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_59a1: + +0.59a1 (2021-08-08) +------------------- + +- The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`) + .. _v0_59a0: 0.59a0 (2021-08-06) From fc4846850fffd54561bc125332dfe97bb41ff42e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 20:21:13 -0700 Subject: [PATCH 0433/1866] New way of deriving named parameters using explain, refs #1421 --- datasette/utils/__init__.py | 12 ++++++++++++ datasette/views/base.py | 1 - datasette/views/database.py | 5 ++++- tests/test_utils.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index aec5a55b..44641a87 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1076,3 +1076,15 @@ class PrefixedUrlString(str): class StartupError(Exception): pass + + +_re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") + +async def derive_named_parameters(db, sql): + explain = 'explain {}'.format(sql.strip().rstrip(";")) + possible_params = _re_named_parameter.findall(sql) + try: + results = await db.execute(explain, {p: None for p in possible_params}) + return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"] + except sqlite3.DatabaseError: + return [] diff --git a/datasette/views/base.py b/datasette/views/base.py index cd584899..1cea1386 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -159,7 +159,6 @@ class BaseView: class DataView(BaseView): name = "" - re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") async def options(self, request, *args, **kwargs): r = Response.text("ok") diff --git a/datasette/views/database.py b/datasette/views/database.py index 29600659..7c36034c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,6 +10,7 @@ import markupsafe from datasette.utils import ( await_me_maybe, check_visibility, + derive_named_parameters, to_css_class, validate_sql_select, is_url, @@ -223,7 +224,9 @@ class QueryView(DataView): await self.check_permission(request, "execute-sql", database) # Extract any :named parameters - named_parameters = named_parameters or self.re_named_parameter.findall(sql) + named_parameters = named_parameters or await derive_named_parameters( + self.ds.get_database(database), sql + ) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters diff --git a/tests/test_utils.py b/tests/test_utils.py index 97b70ee5..e04efb4b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -626,3 +626,18 @@ def test_parse_metadata(content, expected): utils.parse_metadata(content) else: assert utils.parse_metadata(content) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sql,expected", ( + ("select 1", []), + ("select 1 + :one", ["one"]), + ("select 1 + :one + :two", ["one", "two"]), + ("select 'bob' || '0:00' || :cat", ["cat"]), + ("select this is invalid", []), +)) +async def test_derive_named_parameters(sql, expected): + ds = Datasette([], memory=True) + db = ds.get_database("_memory") + params = await utils.derive_named_parameters(db, sql) + assert params == expected From b1fed48a95516ae84c0f020582303ab50ab817e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Aug 2021 20:26:08 -0700 Subject: [PATCH 0434/1866] derive_named_parameters falls back to regex on SQL error, refs #1421 --- datasette/utils/__init__.py | 5 +++-- tests/test_utils.py | 17 ++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 44641a87..70ac8976 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1080,11 +1080,12 @@ class StartupError(Exception): _re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") + async def derive_named_parameters(db, sql): - explain = 'explain {}'.format(sql.strip().rstrip(";")) + explain = "explain {}".format(sql.strip().rstrip(";")) possible_params = _re_named_parameter.findall(sql) try: results = await db.execute(explain, {p: None for p in possible_params}) return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"] except sqlite3.DatabaseError: - return [] + return possible_params diff --git a/tests/test_utils.py b/tests/test_utils.py index e04efb4b..e1b61072 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -629,13 +629,16 @@ def test_parse_metadata(content, expected): @pytest.mark.asyncio -@pytest.mark.parametrize("sql,expected", ( - ("select 1", []), - ("select 1 + :one", ["one"]), - ("select 1 + :one + :two", ["one", "two"]), - ("select 'bob' || '0:00' || :cat", ["cat"]), - ("select this is invalid", []), -)) +@pytest.mark.parametrize( + "sql,expected", + ( + ("select 1", []), + ("select 1 + :one", ["one"]), + ("select 1 + :one + :two", ["one", "two"]), + ("select 'bob' || '0:00' || :cat", ["cat"]), + ("select this is invalid :one, :two, :three", ["one", "two", "three"]), + ), +) async def test_derive_named_parameters(sql, expected): ds = Datasette([], memory=True) db = ds.get_database("_memory") From e837095ef35ae155b4c78cc9a8b7133a48c94f03 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Aug 2021 16:53:23 -0700 Subject: [PATCH 0435/1866] Column metadata, closes #942 --- datasette/static/app.css | 17 ++++++++++++++++- datasette/static/table.js | 9 +++++++++ datasette/templates/_table.html | 2 +- datasette/templates/table.html | 8 ++++++++ datasette/views/table.py | 2 ++ docs/metadata.rst | 28 ++++++++++++++++++++++++++++ tests/fixtures.py | 6 ++++++ tests/test_html.py | 18 ++++++++++++++++++ 8 files changed, 88 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index c6be1e97..bf068fdf 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -784,9 +784,14 @@ svg.dropdown-menu-icon { font-size: 0.7em; color: #666; margin: 0; - padding: 0; padding: 4px 8px 4px 8px; } +.dropdown-menu .dropdown-column-description { + margin: 0; + color: #666; + padding: 4px 8px 4px 8px; + max-width: 20em; +} .dropdown-menu li { border-bottom: 1px solid #ccc; } @@ -836,6 +841,16 @@ svg.dropdown-menu-icon { background-repeat: no-repeat; } +dl.column-descriptions dt { + font-weight: bold; +} +dl.column-descriptions dd { + padding-left: 1.5em; + white-space: pre-wrap; + line-height: 1.1em; + color: #666; +} + .anim-scale-in { animation-name: scale-in; animation-duration: 0.15s; diff --git a/datasette/static/table.js b/datasette/static/table.js index 991346df..85bf073f 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -9,6 +9,7 @@ var DROPDOWN_HTML = `
  • Show not-blank rows

    • +

    `; var DROPDOWN_ICON_SVG = ` @@ -166,6 +167,14 @@ var DROPDOWN_ICON_SVG = `
    {% for column in display_columns %} - + {% if not column.sortable %} {{ column.name }} {% else %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 211352b5..466e8a47 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -51,6 +51,14 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} +{% if metadata.columns %} +
    + {% for column_name, column_description in metadata.columns.items() %} +
    {{ column_name }}
    {{ column_description }}
    + {% endfor %} +
    +{% endif %} + {% if filtered_table_rows_count or human_description_en %}

    {% if filtered_table_rows_count or filtered_table_rows_count == 0 %}{{ "{:,}".format(filtered_table_rows_count) }} row{% if filtered_table_rows_count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %} diff --git a/datasette/views/table.py b/datasette/views/table.py index 456d8069..486a6131 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -125,6 +125,7 @@ class RowTableShared(DataView): """Returns columns, rows for specified table - including fancy foreign key treatment""" db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) + column_descriptions = table_metadata.get("columns") or {} column_details = {col.name: col for col in await db.table_column_details(table)} sortable_columns = await self.sortable_columns_for_table(database, table, True) pks = await db.primary_keys(table) @@ -147,6 +148,7 @@ class RowTableShared(DataView): "is_pk": r[0] in pks_for_display, "type": type_, "notnull": notnull, + "description": column_descriptions.get(r[0]), } ) diff --git a/docs/metadata.rst b/docs/metadata.rst index dad5adca..35b8aede 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -78,6 +78,34 @@ The three visible metadata fields you can apply to everything, specific database For each of these you can provide just the ``*_url`` field and Datasette will treat that as the default link label text and display the URL directly on the page. +.. _metadata_column_descriptions: + +Column descriptions +------------------- + +You can include descriptions for your columns by adding a ``"columns": {"name-of-column": "description-of-column"}`` block to your table metadata: + +.. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "columns": { + "column1": "Description of column 1", + "column2": "Description of column 2" + } + } + } + } + } + } + +These will be displayed at the top of the table page, and will also show in the cog menu for each column. + +You can see an example of how these look at `latest.datasette.io/fixtures/roadside_attractions `__. + Specifying units for a column ----------------------------- diff --git a/tests/fixtures.py b/tests/fixtures.py index 880e4347..4a420e4b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -336,6 +336,12 @@ METADATA = { "fts_table": "searchable_fts", "fts_pk": "pk", }, + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, "attraction_characteristic": {"sort_desc": "pk"}, "facet_cities": {"sort": "name"}, "paginated_view": {"size": 25}, diff --git a/tests/test_html.py b/tests/test_html.py index b1b6c1f3..f12f89cd 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1777,3 +1777,21 @@ def test_trace_correctly_escaped(app_client): response = app_client.get("/fixtures?sql=select+'

    Hello'&_trace=1") assert "select '

    Hello" not in response.text assert "select '<h1>Hello" in response.text + + +def test_column_metadata(app_client): + response = app_client.get("/fixtures/roadside_attractions") + soup = Soup(response.body, "html.parser") + dl = soup.find("dl") + assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [ + ("name", "The name of the attraction"), + ("address", "The street address for the attraction"), + ] + assert ( + soup.select("th[data-column=name]")[0]["data-column-description"] + == "The name of the attraction" + ) + assert ( + soup.select("th[data-column=address]")[0]["data-column-description"] + == "The street address for the attraction" + ) From 77f46297a88ac7e49dad2139410b01ee56d5f99c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Aug 2021 18:01:57 -0700 Subject: [PATCH 0436/1866] Rename --help-config to --help-settings, closes #1431 --- datasette/cli.py | 12 ++++++------ docs/datasette-serve-help.txt | 2 +- tests/test_cli.py | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index e53f3d8e..d4e23c70 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -51,7 +51,7 @@ class Config(click.ParamType): name, value = config.split(":", 1) if name not in DEFAULT_SETTINGS: self.fail( - f"{name} is not a valid option (--help-config to see all)", + f"{name} is not a valid option (--help-settings to see all)", param, ctx, ) @@ -84,7 +84,7 @@ class Setting(CompositeParamType): name, value = config if name not in DEFAULT_SETTINGS: self.fail( - f"{name} is not a valid option (--help-config to see all)", + f"{name} is not a valid option (--help-settings to see all)", param, ctx, ) @@ -408,7 +408,7 @@ def uninstall(packages, yes): help="Run an HTTP GET request against this path, print results and exit", ) @click.option("--version-note", help="Additional note to show on /-/versions") -@click.option("--help-config", is_flag=True, help="Show available config options") +@click.option("--help-settings", is_flag=True, help="Show available settings") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") @click.option( "-o", @@ -456,7 +456,7 @@ def serve( root, get, version_note, - help_config, + help_settings, pdb, open_browser, create, @@ -466,9 +466,9 @@ def serve( return_instance=False, ): """Serve up specified SQLite database files with a web UI""" - if help_config: + if help_settings: formatter = formatting.HelpFormatter() - with formatter.section("Config options"): + with formatter.section("Settings"): formatter.write_dl( [ (option.name, f"{option.help} (default={option.default})") diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index ec3f41a0..2911977a 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -32,7 +32,7 @@ Options: --get TEXT Run an HTTP GET request against this path, print results and exit --version-note TEXT Additional note to show on /-/versions - --help-config Show available config options + --help-settings Show available settings --pdb Launch debugger on any errors -o, --open Open Datasette in your web browser --create Create database files if they do not exist diff --git a/tests/test_cli.py b/tests/test_cli.py index e31a305e..763fe2e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,7 @@ from .fixtures import ( EXPECTED_PLUGINS, ) import asyncio +from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve from datasette.version import __version__ @@ -147,7 +148,7 @@ def test_metadata_yaml(): root=False, version_note=None, get=None, - help_config=False, + help_settings=False, pdb=False, crossdb=False, open_browser=False, @@ -291,3 +292,10 @@ def test_weird_database_names(ensure_eventloop, tmpdir, filename): cli, [db_path, "--get", "/{}".format(urllib.parse.quote(filename_no_stem))] ) assert result2.exit_code == 0, result2.output + + +def test_help_settings(): + runner = CliRunner() + result = runner.invoke(cli, ["--help-settings"]) + for setting in SETTINGS: + assert setting.name in result.output From ca4f83dc7b1d573b92a8921fca96d3ed490614c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 12 Aug 2021 18:10:36 -0700 Subject: [PATCH 0437/1866] Rename config= to settings=, refs #1432 --- datasette/app.py | 8 ++++---- datasette/cli.py | 8 ++++---- datasette/templates/table.html | 2 +- datasette/views/base.py | 2 +- datasette/views/database.py | 2 +- tests/fixtures.py | 20 ++++++++++---------- tests/test_api.py | 8 ++++---- tests/test_custom_pages.py | 2 +- tests/test_facets.py | 2 +- tests/test_html.py | 14 ++++++++------ 10 files changed, 35 insertions(+), 33 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f2f75884..8cbaaf9f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -200,7 +200,7 @@ class Datasette: plugins_dir=None, static_mounts=None, memory=False, - config=None, + settings=None, secret=None, version_note=None, config_dir=None, @@ -279,7 +279,7 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not config: config = json.loads((config_dir / "settings.json").read_text()) - self._settings = dict(DEFAULT_SETTINGS, **(config or {})) + self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note self.executor = futures.ThreadPoolExecutor( @@ -419,8 +419,8 @@ class Datasette: def setting(self, key): return self._settings.get(key, None) - def config_dict(self): - # Returns a fully resolved config dictionary, useful for templates + def settings_dict(self): + # Returns a fully resolved settings dictionary, useful for templates return {option.name: self.setting(option.name) for option in SETTINGS} def _metadata_recursive_update(self, orig, updated): diff --git a/datasette/cli.py b/datasette/cli.py index d4e23c70..ea6da748 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -495,14 +495,14 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) - combined_config = {} + combined_settings = {} if config: click.echo( "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", err=True, ) - combined_config.update(config) - combined_config.update(settings) + combined_settings.update(config) + combined_settings.update(settings) kwargs = dict( immutables=immutable, @@ -514,7 +514,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - config=combined_config, + settings=combined_settings, memory=memory, secret=secret, version_note=version_note, diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 466e8a47..a28945ad 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -201,7 +201,7 @@ CSV options: {% if expandable_columns %}{% endif %} - {% if next_url and config.allow_csv_stream %}{% endif %} + {% if next_url and settings.allow_csv_stream %}{% endif %} {% for key, value in url_csv_hidden_args %} diff --git a/datasette/views/base.py b/datasette/views/base.py index 1cea1386..3333781c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -614,7 +614,7 @@ class DataView(BaseView): ] + [("_size", "max")], "datasette_version": __version__, - "config": self.ds.config_dict(), + "settings": self.ds.settings_dict(), }, } if "metadata" not in context: diff --git a/datasette/views/database.py b/datasette/views/database.py index 7c36034c..e3070ce6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -456,7 +456,7 @@ class QueryView(DataView): "canned_query": canned_query, "edit_sql_url": edit_sql_url, "metadata": metadata, - "config": self.ds.config_dict(), + "settings": self.ds.settings_dict(), "request": request, "show_hide_link": show_hide_link, "show_hide_text": show_hide_text, diff --git a/tests/fixtures.py b/tests/fixtures.py index 4a420e4b..dc22c609 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -99,7 +99,7 @@ def make_app_client( max_returned_rows=None, cors=False, memory=False, - config=None, + settings=None, filename="fixtures.db", is_immutable=False, extra_databases=None, @@ -129,7 +129,7 @@ def make_app_client( # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) - config = config or {} + settings = settings or {} for key, value in { "default_page_size": 50, "max_returned_rows": max_returned_rows or 100, @@ -138,8 +138,8 @@ def make_app_client( # errors when running the full test suite: "num_sql_threads": 1, }.items(): - if key not in config: - config[key] = value + if key not in settings: + settings[key] = value ds = Datasette( files, immutables=immutables, @@ -147,7 +147,7 @@ def make_app_client( cors=cors, metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, - config=config, + settings=settings, inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, @@ -171,7 +171,7 @@ def app_client_no_files(): @pytest.fixture(scope="session") def app_client_base_url_prefix(): - with make_app_client(config={"base_url": "/prefix/"}) as client: + with make_app_client(settings={"base_url": "/prefix/"}) as client: yield client @@ -210,13 +210,13 @@ def app_client_two_attached_databases_one_immutable(): @pytest.fixture(scope="session") def app_client_with_hash(): - with make_app_client(config={"hash_urls": True}, is_immutable=True) as client: + with make_app_client(settings={"hash_urls": True}, is_immutable=True) as client: yield client @pytest.fixture(scope="session") def app_client_with_trace(): - with make_app_client(config={"trace_debug": True}, is_immutable=True) as client: + with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client: yield client @@ -234,13 +234,13 @@ def app_client_returned_rows_matches_page_size(): @pytest.fixture(scope="session") def app_client_larger_cache_size(): - with make_app_client(config={"cache_size_kb": 2500}) as client: + with make_app_client(settings={"cache_size_kb": 2500}) as client: yield client @pytest.fixture(scope="session") def app_client_csv_max_mb_one(): - with make_app_client(config={"max_csv_mb": 1}) as client: + with make_app_client(settings={"max_csv_mb": 1}) as client: yield client diff --git a/tests/test_api.py b/tests/test_api.py index 83cca521..1e93c62e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1711,14 +1711,14 @@ def test_suggested_facets(app_client): def test_allow_facet_off(): - with make_app_client(config={"allow_facet": False}) as client: + with make_app_client(settings={"allow_facet": False}) as client: assert 400 == client.get("/fixtures/facetable.json?_facet=planet_int").status # Should not suggest any facets either: assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] def test_suggest_facets_off(): - with make_app_client(config={"suggest_facets": False}) as client: + with make_app_client(settings={"suggest_facets": False}) as client: # Now suggested_facets should be [] assert [] == client.get("/fixtures/facetable.json").json["suggested_facets"] @@ -1883,7 +1883,7 @@ def test_config_cache_size(app_client_larger_cache_size): def test_config_force_https_urls(): - with make_app_client(config={"force_https_urls": True}) as client: + with make_app_client(settings={"force_https_urls": True}) as client: response = client.get("/fixtures/facetable.json?_size=3&_facet=state") assert response.json["next_url"].startswith("https://") assert response.json["facet_results"]["state"]["results"][0][ @@ -1921,7 +1921,7 @@ def test_custom_query_with_unicode_characters(app_client): @pytest.mark.parametrize("trace_debug", (True, False)) def test_trace(trace_debug): - with make_app_client(config={"trace_debug": trace_debug}) as client: + with make_app_client(settings={"trace_debug": trace_debug}) as client: response = client.get("/fixtures/simple_primary_key.json?_trace=1") assert response.status == 200 diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 5a71f56d..76c67397 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -14,7 +14,7 @@ def custom_pages_client(): @pytest.fixture(scope="session") def custom_pages_client_with_base_url(): with make_app_client( - template_dir=TEST_TEMPLATE_DIRS, config={"base_url": "/prefix/"} + template_dir=TEST_TEMPLATE_DIRS, settings={"base_url": "/prefix/"} ) as client: yield client diff --git a/tests/test_facets.py b/tests/test_facets.py index 18fb8c3b..22927512 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -351,7 +351,7 @@ async def test_json_array_with_blanks_and_nulls(): @pytest.mark.asyncio async def test_facet_size(): - ds = Datasette([], memory=True, config={"max_returned_rows": 50}) + ds = Datasette([], memory=True, settings={"max_returned_rows": 50}) db = ds.add_database(Database(ds, memory_name="test_facet_size")) await db.execute_write( "create table neighbourhoods(city text, neighbourhood text)", block=True diff --git a/tests/test_html.py b/tests/test_html.py index f12f89cd..90fcdae7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -214,7 +214,7 @@ def test_definition_sql(path, expected_definition_sql, app_client): def test_table_cell_truncation(): - with make_app_client(config={"truncate_cells_html": 5}) as client: + with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -239,7 +239,7 @@ def test_table_cell_truncation(): def test_row_page_does_not_truncate(): - with make_app_client(config={"truncate_cells_html": 5}) as client: + with make_app_client(settings={"truncate_cells_html": 5}) as client: response = client.get("/fixtures/facetable/1") assert response.status == 200 table = Soup(response.body, "html.parser").find("table") @@ -1072,7 +1072,9 @@ def test_database_download_disallowed_for_memory(): def test_allow_download_off(): - with make_app_client(is_immutable=True, config={"allow_download": False}) as client: + with make_app_client( + is_immutable=True, settings={"allow_download": False} + ) as client: response = client.get("/fixtures") soup = Soup(response.body, "html.parser") assert not len(soup.findAll("a", {"href": re.compile(r"\.db$")})) @@ -1486,7 +1488,7 @@ def test_query_error(app_client): def test_config_template_debug_on(): - with make_app_client(config={"template_debug": True}) as client: + with make_app_client(settings={"template_debug": True}) as client: response = client.get("/fixtures/facetable?_context=1") assert response.status == 200 assert response.text.startswith("
    {")
@@ -1500,7 +1502,7 @@ def test_config_template_debug_off(app_client):
 
 def test_debug_context_includes_extra_template_vars():
     # https://github.com/simonw/datasette/issues/693
-    with make_app_client(config={"template_debug": True}) as client:
+    with make_app_client(settings={"template_debug": True}) as client:
         response = client.get("/fixtures/facetable?_context=1")
         # scope_path is added by PLUGIN1
         assert "scope_path" in response.text
@@ -1744,7 +1746,7 @@ def test_facet_more_links(
     expected_ellipses_url,
 ):
     with make_app_client(
-        config={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
+        settings={"max_returned_rows": max_returned_rows, "default_facet_size": 2}
     ) as client:
         response = client.get(path)
         soup = Soup(response.body, "html.parser")

From bbc4756f9e8180c7a40c57f8a35e39dee7be7807 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 12 Aug 2021 20:54:25 -0700
Subject: [PATCH 0438/1866] Settings fix, refs #1433

---
 datasette/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index 8cbaaf9f..adc543ef 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -277,7 +277,7 @@ class Datasette:
         self.static_mounts = static_mounts or []
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
-        if config_dir and (config_dir / "settings.json").exists() and not config:
+        if config_dir and (config_dir / "settings.json").exists() and not settings:
             config = json.loads((config_dir / "settings.json").read_text())
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions

From 2883098770fc66e50183b2b231edbde20848d4d6 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 12 Aug 2021 22:10:07 -0700
Subject: [PATCH 0439/1866] Fixed config_dir mode, refs #1432

---
 datasette/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/app.py b/datasette/app.py
index adc543ef..06db740e 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -278,7 +278,7 @@ class Datasette:
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
         if config_dir and (config_dir / "settings.json").exists() and not settings:
-            config = json.loads((config_dir / "settings.json").read_text())
+            settings = json.loads((config_dir / "settings.json").read_text())
         self._settings = dict(DEFAULT_SETTINGS, **(settings or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note

From adb5b70de5cec3c3dd37184defe606a082c232cf Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 16 Aug 2021 11:56:32 -0700
Subject: [PATCH 0440/1866] Show count of facet values if ?_facet_size=max,
 closes #1423

---
 datasette/static/app.css       |  5 +++++
 datasette/templates/table.html |  4 +++-
 datasette/views/table.py       |  1 +
 tests/test_html.py             | 22 +++++++++++++++++++++-
 4 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/datasette/static/app.css b/datasette/static/app.css
index bf068fdf..af3e14d5 100644
--- a/datasette/static/app.css
+++ b/datasette/static/app.css
@@ -633,6 +633,11 @@ form button[type=button] {
     width: 250px;
     margin-right: 15px;
 }
+.facet-info-total {
+    font-size: 0.8em;
+    color: #666;
+    padding-right: 0.25em;
+}
 .facet-info li,
 .facet-info ul {
     margin: 0;
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index a28945ad..6ba301b5 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -156,7 +156,9 @@
         {% for facet_info in sorted_facet_results %}
             
    
                 
    
-                    {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                    {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                        {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %}
+                    
                     {% if facet_info.hideable %}
                         ✖
                     {% endif %}
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 486a6131..83f7c7cb 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -928,6 +928,7 @@ class TableView(RowTableShared):
                     key=lambda f: (len(f["results"]), f["name"]),
                     reverse=True,
                 ),
+                "show_facet_counts": special_args.get("_facet_size") == "max",
                 "extra_wheres_for_ui": extra_wheres_for_ui,
                 "form_hidden_args": form_hidden_args,
                 "is_sortable": any(c["sortable"] for c in display_columns),
diff --git a/tests/test_html.py b/tests/test_html.py
index 90fcdae7..e73ccd2f 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -479,7 +479,7 @@ def test_facet_display(app_client):
     for div in divs:
         actual.append(
             {
-                "name": div.find("strong").text,
+                "name": div.find("strong").text.split()[0],
                 "items": [
                     {
                         "name": a.text,
@@ -1797,3 +1797,23 @@ def test_column_metadata(app_client):
         soup.select("th[data-column=address]")[0]["data-column-description"]
         == "The street address for the attraction"
     )
+
+
+@pytest.mark.parametrize("use_facet_size_max", (True, False))
+def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
+    # https://github.com/simonw/datasette/issues/1423
+    with make_app_client(settings={"max_returned_rows": 100}) as client:
+        path = "/fixtures/sortable?_facet=content&_facet=pk1"
+        if use_facet_size_max:
+            path += "&_facet_size=max"
+        response = client.get(path)
+        assert response.status == 200
+    fragments = (
+        '>100',
+        '8',
+    )
+    for fragment in fragments:
+        if use_facet_size_max:
+            assert fragment in response.text
+        else:
+            assert fragment not in response.text

From d84e574e59c51ddcd6cf60a6f9b3d45182daf824 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:09:38 -0700
Subject: [PATCH 0441/1866] Ability to deploy demos of branches

* Ability to deploy additional branch demos, closes #1442
* Only run tests before deploy on main branch
* Documentation for continuous deployment
---
 .github/workflows/deploy-latest.yml |  8 +++++++-
 docs/contributing.rst               | 11 +++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 849adb40..1a07503a 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -29,6 +29,7 @@ jobs:
         python -m pip install -e .[docs]
         python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
@@ -50,6 +51,8 @@ jobs:
       run: |-
         gcloud config set run/region us-central1
         gcloud config set project datasette-222320
+        export SUFFIX="-${GITHUB_REF#refs/heads/}"
+        export SUFFIX=${SUFFIX#-main}
         datasette publish cloudrun fixtures.db extra_database.db \
             -m fixtures.json \
             --plugins-dir=plugins \
@@ -57,7 +60,10 @@ jobs:
             --version-note=$GITHUB_SHA \
             --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \
             --install=pysqlite3-binary \
-            --service=datasette-latest
+            --service "datasette-latest$SUFFIX"
+    - name: Deploy to docs as well (only for main)
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |-
         # Deploy docs.db to a different service
         datasette publish cloudrun docs.db \
             --branch=$GITHUB_SHA \
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 8a638e0b..07f2a0e4 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -202,6 +202,17 @@ For added productivity, you can use use `sphinx-autobuild `__ is re-deployed automatically to Google Cloud Run for every push to ``main`` that passes the test suite. This is implemented by the GitHub Actions workflow at `.github/workflows/deploy-latest.yml `__.
+
+Specific branches can also be set to automatically deploy by adding them to the ``on: push: branches`` block at the top of the workflow YAML file. Branches configured in this way will be deployed to a new Cloud Run service whether or not their tests pass.
+
+The Cloud Run URL for a branch demo can be found in the GitHub Actions logs.
+
 .. _contributing_release:
 
 Release process

From 4eb3ae40fb223a66ae574fb84fac99e96183b08d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:17:44 -0700
Subject: [PATCH 0442/1866] Don't bother building docs if not on main

Refs ##1442
---
 .github/workflows/deploy-latest.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml
index 1a07503a..1ae96e89 100644
--- a/.github/workflows/deploy-latest.yml
+++ b/.github/workflows/deploy-latest.yml
@@ -36,6 +36,7 @@ jobs:
     - name: Build fixtures.db
       run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db
     - name: Build docs.db
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         sphinx-build -b xml . _build

From 7e15422aacfa9e9735cb9f9beaa32250edbf4905 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 19 Aug 2021 14:23:43 -0700
Subject: [PATCH 0443/1866] Documentation for datasette.databases property,
 closes #1443

---
 docs/internals.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index 058a8969..d5db7ffa 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -196,6 +196,17 @@ Datasette class
 
 This object is an instance of the ``Datasette`` class, passed to many plugin hooks as an argument called ``datasette``.
 
+.. _datasette_databases:
+
+.databases
+----------
+
+Property exposing an ordered dictionary of databases currently connected to Datasette.
+
+The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances.
+
+All databases are listed, irrespective of user permissions. This means that the ``_internal`` database will always be listed here.
+
 .. _datasette_plugin_config:
 
 .plugin_config(plugin_name, database=None, table=None)

From 92a99d969c01633dba14cceebeda65daaedaec17 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Tue, 24 Aug 2021 11:13:42 -0700
Subject: [PATCH 0444/1866] Added not-footer wrapper div, refs #1446

---
 datasette/templates/base.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/datasette/templates/base.html b/datasette/templates/base.html
index e61edc4f..c9aa7e31 100644
--- a/datasette/templates/base.html
+++ b/datasette/templates/base.html
@@ -13,6 +13,7 @@
 {% block extra_head %}{% endblock %}
 
 
+
    
 
    
 
    {% block footer %}{% include "_footer.html" %}{% endblock %}
    
 
 {% include "_close_open_menus.html" %}

From 93c3a7ffbfb3378f743ebce87d033cf1ce7689e0 Mon Sep 17 00:00:00 2001
From: Tim Sherratt 
Date: Wed, 25 Aug 2021 11:28:58 +1000
Subject: [PATCH 0445/1866] Remove underscore from search mode parameter name
 (#1447)

The text refers to the parameter as `searchmode` but the `metadata.json` example uses `search_mode`. The latter doesn't actually seem to work.
---
 docs/full_text_search.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst
index f549296f..90b2e8c1 100644
--- a/docs/full_text_search.rst
+++ b/docs/full_text_search.rst
@@ -70,7 +70,7 @@ Here is an example which enables full-text search (with SQLite advanced search o
                     "display_ads": {
                         "fts_table": "ads_fts",
                         "fts_pk": "id",
-                        "search_mode": "raw"
+                        "searchmode": "raw"
                     }
                 }
             }

From 5161422b7fa249c6b7d6dc47ec6f483d3fdbd170 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Aug 2021 18:29:26 -0700
Subject: [PATCH 0446/1866] Update trustme requirement from <0.9,>=0.7 to
 >=0.7,<0.10 (#1433)

Updates the requirements on [trustme](https://github.com/python-trio/trustme) to permit the latest version.
- [Release notes](https://github.com/python-trio/trustme/releases)
- [Commits](https://github.com/python-trio/trustme/compare/v0.7.0...v0.9.0)

---
updated-dependencies:
- dependency-name: trustme
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 65e99848..a3866515 100644
--- a/setup.py
+++ b/setup.py
@@ -73,7 +73,7 @@ setup(
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.6b0",
             "pytest-timeout>=1.4.2,<1.5",
-            "trustme>=0.7,<0.9",
+            "trustme>=0.7,<0.10",
         ],
         "rich": ["rich"],
     },

From a1a33bb5822214be1cebd98cd858b2058d91a4aa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 24 Aug 2021 18:29:55 -0700
Subject: [PATCH 0447/1866] Bump black from 21.6b0 to 21.7b0 (#1400)

Bumps [black](https://github.com/psf/black) from 21.6b0 to 21.7b0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a3866515..84f32087 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.4",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
-            "black==21.6b0",
+            "black==21.7b0",
             "pytest-timeout>=1.4.2,<1.5",
             "trustme>=0.7,<0.10",
         ],

From 3655bb49a464bcc8004e491cc4d4de292f1acd62 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 17:48:54 -0700
Subject: [PATCH 0448/1866] Better default help text, closes #1450

---
 datasette/cli.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index ea6da748..65da5613 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -123,7 +123,11 @@ def sqlite_extensions(fn):
 @click.version_option(version=__version__)
 def cli():
     """
-    Datasette!
+    Datasette is an open source multi-tool for exploring and publishing data
+
+    \b
+    About Datasette: https://datasette.io/
+    Full documentation: https://docs.datasette.io/
     """
 
 

From 30c18576d603366dc3bd83ba50de1b7e70844430 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 18:39:42 -0700
Subject: [PATCH 0449/1866] register_commands() plugin hook, closes #1449

---
 datasette/cli.py       |  3 +++
 datasette/hookspecs.py |  5 ++++
 docs/plugin_hooks.rst  | 45 +++++++++++++++++++++++++++++++++
 tests/test_plugins.py  | 57 +++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 109 insertions(+), 1 deletion(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 65da5613..22e2338a 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -595,6 +595,9 @@ def serve(
     uvicorn.run(ds.app(), **uvicorn_kwargs)
 
 
+pm.hook.register_commands(cli=cli)
+
+
 async def check_databases(ds):
     # Run check_connection against every connected database
     # to confirm they are all usable
diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py
index 56c79d23..1d4e3b27 100644
--- a/datasette/hookspecs.py
+++ b/datasette/hookspecs.py
@@ -79,6 +79,11 @@ def register_routes(datasette):
     """Register URL routes: return a list of (regex, view_function) pairs"""
 
 
+@hookspec
+def register_commands(cli):
+    """Register additional CLI commands, e.g. 'datasette mycommand ...'"""
+
+
 @hookspec
 def actor_from_request(datasette, request):
     """Return an actor dictionary based on the incoming request"""
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index 5cdb1623..a6fe1071 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -587,6 +587,51 @@ See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes u
 
 Examples: `datasette-auth-github `__, `datasette-psutil `__
 
+.. _plugin_register_commands:
+
+register_commands(cli)
+----------------------
+
+``cli`` - the root Datasette `Click command group `__
+    Use this to register additional CLI commands
+
+Register additional CLI commands that can be run using ``datsette yourcommand ...``. This provides a mechanism by which plugins can add new CLI commands to Datasette.
+
+This example registers a new ``datasette verify file1.db file2.db`` command that checks if the provided file paths are valid SQLite databases:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import click
+    import sqlite3
+
+    @hookimpl
+    def register_commands(cli):
+        @cli.command()
+        @click.argument("files", type=click.Path(exists=True), nargs=-1)
+        def verify(files):
+            "Verify that files can be opened by Datasette"
+            for file in files:
+                conn = sqlite3.connect(str(file))
+                try:
+                    conn.execute("select * from sqlite_master")
+                except sqlite3.DatabaseError:
+                    raise click.ClickException("Invalid database: {}".format(file))
+
+The new command can then be executed like so::
+
+    datasette verify fixtures.db
+
+Help text (from the docstring for the function plus any defined Click arguments or options) will become available using::
+
+    datasette verify --help
+
+Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator.Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
+
+Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-dir mechanism ` - they need to be installed into the same virtual environment as Datasette using ``pip install``. Provided it has a ``setup.py`` file (see :ref:`writing_plugins_packaging`) you can run ``pip install`` directly against the directory in which you are developing your plugin like so::
+
+    pip install -e path/to/my/datasette-plugin
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index ec8ff0c5..a024c39b 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -6,13 +6,15 @@ from .fixtures import (
     TEMP_PLUGIN_SECRET_FILE,
     TestClient as _TestClient,
 )  # noqa
+from click.testing import CliRunner
 from datasette.app import Datasette
-from datasette import cli
+from datasette import cli, hookimpl
 from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
 from datasette.utils.sqlite import sqlite3
 from datasette.utils import CustomRow
 from jinja2.environment import Template
 import base64
+import importlib
 import json
 import os
 import pathlib
@@ -902,3 +904,56 @@ def test_hook_get_metadata(app_client):
     assert "Hello from local metadata" == meta["databases"]["from-local"]["title"]
     assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"]
     pm.hook.get_metadata = og_pm_hook_get_metadata
+
+
+def _extract_commands(output):
+    lines = output.split("Commands:\n", 1)[1].split("\n")
+    return {line.split()[0].replace("*", "") for line in lines if line.strip()}
+
+
+def test_hook_register_commands():
+    # Without the plugin should have seven commands
+    runner = CliRunner()
+    result = runner.invoke(cli.cli, "--help")
+    commands = _extract_commands(result.output)
+    assert commands == {
+        "serve",
+        "inspect",
+        "install",
+        "package",
+        "plugins",
+        "publish",
+        "uninstall",
+    }
+
+    # Now install a plugin
+    class VerifyPlugin:
+        __name__ = "VerifyPlugin"
+
+        @hookimpl
+        def register_commands(self, cli):
+            @cli.command()
+            def verify():
+                pass
+
+            @cli.command()
+            def unverify():
+                pass
+
+    pm.register(VerifyPlugin(), name="verify")
+    importlib.reload(cli)
+    result2 = runner.invoke(cli.cli, "--help")
+    commands2 = _extract_commands(result2.output)
+    assert commands2 == {
+        "serve",
+        "inspect",
+        "install",
+        "package",
+        "plugins",
+        "publish",
+        "uninstall",
+        "verify",
+        "unverify",
+    }
+    pm.unregister(name="verify")
+    importlib.reload(cli)

From d3ea36713194e3d92ed4c066337400146c921d0e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 27 Aug 2021 18:55:54 -0700
Subject: [PATCH 0450/1866] Release 0.59a2

Refs #942, #1421, #1423, #1431, #1443, #1446, #1449
---
 datasette/version.py  |  2 +-
 docs/changelog.rst    | 13 +++++++++++++
 docs/plugin_hooks.rst |  2 +-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index f5fbfb3f..87b18fab 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59a1"
+__version__ = "0.59a2"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 1406a7ca..737a151b 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,19 @@
 Changelog
 =========
 
+.. _v0_59a2:
+
+0.59a2 (2021-08-27)
+-------------------
+
+- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
+- New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
+- Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
+- Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
+- Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)
+- ``datasette.databases`` property is now a documented API. (:issue:`1443`)
+- Datasette base template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
+
 .. _v0_59a1:
 
 0.59a1 (2021-08-08)
diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index a6fe1071..be1624a3 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -587,7 +587,7 @@ See :ref:`writing_plugins_designing_urls` for tips on designing the URL routes u
 
 Examples: `datasette-auth-github `__, `datasette-psutil `__
 
-.. _plugin_register_commands:
+.. _plugin_hook_register_commands:
 
 register_commands(cli)
 ----------------------

From 50c35b66a476c186549167140b6ebc0a6ec43fc1 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 28 Aug 2021 04:14:38 -0700
Subject: [PATCH 0451/1866] Added missing space

---
 docs/plugin_hooks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index be1624a3..d61dc727 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -626,7 +626,7 @@ Help text (from the docstring for the function plus any defined Click arguments
 
     datasette verify --help
 
-Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator.Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
+Plugins can register multiple commands by making multiple calls to the ``@cli.command()`` decorator. Consult the `Click documentation `__ for full details on how to build a CLI command, including how to define arguments and options.
 
 Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-dir mechanism ` - they need to be installed into the same virtual environment as Datasette using ``pip install``. Provided it has a ``setup.py`` file (see :ref:`writing_plugins_packaging`) you can run ``pip install`` directly against the directory in which you are developing your plugin like so::
 

From 67cbf0ae7243431bf13702e6e3ba466b619c4d6f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 28 Aug 2021 04:17:03 -0700
Subject: [PATCH 0452/1866] Example for register_commands, refs #1449

---
 docs/plugin_hooks.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst
index d61dc727..23f19e38 100644
--- a/docs/plugin_hooks.rst
+++ b/docs/plugin_hooks.rst
@@ -80,7 +80,6 @@ You can now use this filter in your custom templates like so::
 
     Table name: {{ table|uppercase }}
 
-
 .. _plugin_hook_extra_template_vars:
 
 extra_template_vars(template, database, table, columns, view_name, request, datasette)
@@ -632,6 +631,8 @@ Note that ``register_commands()`` plugins cannot used with the :ref:`--plugins-d
 
     pip install -e path/to/my/datasette-plugin
 
+Example: `datasette-verify `_
+
 .. _plugin_register_facet_classes:
 
 register_facet_classes()

From 772f9a07ce363869e0aaa7600617454dc00e6966 Mon Sep 17 00:00:00 2001
From: Robert Gieseke 
Date: Sat, 4 Sep 2021 18:31:38 +0200
Subject: [PATCH 0453/1866] Add scientists to target groups (#1455)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 55160afe..95d430cc 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 
 Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.
 
-Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world.
+Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists and anyone else who has data that they wish to share with the world.
 
 [Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch).
 

From d57ab156b35ec642549fb69d08279850065027d2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 4 Sep 2021 09:33:20 -0700
Subject: [PATCH 0454/1866] Added researchers too, refs #1455

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 95d430cc..ee9d9a5a 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 
 Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.
 
-Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists and anyone else who has data that they wish to share with the world.
+Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world.
 
 [Explore a demo](https://global-power-plants.datasettes.com/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out by [uploading and publishing your own CSV data](https://docs.datasette.io/en/stable/getting_started.html#try-datasette-without-installing-anything-using-glitch).
 

From b28b6cd2fe97f7e193a235877abeec2c8eb0a821 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 12 Sep 2021 13:13:52 -0700
Subject: [PATCH 0455/1866] Warn that execute_write_fn(fn) should be a
 non-async function

---
 docs/internals.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index d5db7ffa..910f2c71 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -664,6 +664,10 @@ This method works like ``.execute_write()``, but instead of a SQL statement you
 
 The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection as long as it is executing.
 
+.. warning::
+
+    ``fn`` needs to be a regular function, not an ``async def`` function.
+
 For example:
 
 .. code-block:: python

From 63886178a649586b403966a27a45881709d2b868 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 22 Sep 2021 15:44:28 -0700
Subject: [PATCH 0456/1866] Describe a common mistake using csrftoken()

---
 docs/internals.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/internals.rst b/docs/internals.rst
index 910f2c71..411327eb 100644
--- a/docs/internals.rst
+++ b/docs/internals.rst
@@ -793,6 +793,10 @@ If your plugin implements a ``
    `` anywhere you will need to i
 
     
 
+If you are rendering templates using the :ref:`datasette_render_template` method the ``csrftoken()`` helper will only work if you provide the ``request=`` argument to that method. If you forget to do this you will see the following error::
+
+    form-urlencoded POST field did not match cookie
+
 You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csrf` hook.
 
 .. _internals_internal:

From 98dcabccbbf9c0800efa74df9b7d1fee81c3cd0c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 8 Oct 2021 17:32:11 -0700
Subject: [PATCH 0457/1866] asyncio_run helper to deal with a 3.10 warning,
 refs #1482

---
 datasette/cli.py            | 13 +++++--------
 datasette/utils/__init__.py |  9 +++++++++
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 22e2338a..95ee9495 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -14,6 +14,7 @@ from runpy import run_module
 import webbrowser
 from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
 from .utils import (
+    asyncio_run,
     StartupError,
     check_connection,
     find_spatialite,
@@ -136,9 +137,7 @@ def cli():
 @click.option("--inspect-file", default="-")
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
-    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
-    loop = asyncio.get_event_loop()
-    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
+    inspect_data = asyncio_run(inspect_(files, sqlite_extensions))
     if inspect_file == "-":
         sys.stdout.write(json.dumps(inspect_data, indent=2))
     else:
@@ -555,10 +554,10 @@ def serve(
         return ds
 
     # Run the "startup" plugin hooks
-    asyncio.get_event_loop().run_until_complete(ds.invoke_startup())
+    asyncio_run(ds.invoke_startup())
 
     # Run async soundness checks - but only if we're not under pytest
-    asyncio.get_event_loop().run_until_complete(check_databases(ds))
+    asyncio_run(check_databases(ds))
 
     if get:
         client = TestClient(ds)
@@ -578,9 +577,7 @@ def serve(
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage
-            path = asyncio.get_event_loop().run_until_complete(
-                initial_path_for_datasette(ds)
-            )
+            path = asyncio_run(initial_path_for_datasette(ds))
             url = f"http://{host}:{port}{path}"
         webbrowser.open(url)
     uvicorn_kwargs = dict(
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 70ac8976..1b7bfe05 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,3 +1089,12 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
+
+
+def asyncio_run(coro):
+    if hasattr(asyncio, "run"):
+        # Added in Python 3.7
+        return asyncio.run(coro)
+    else:
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(coro)

From 1163da89163f357003007b14f458a2c28f2e0a8e Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 8 Oct 2021 17:32:52 -0700
Subject: [PATCH 0458/1866] Update test to handle Python 3.10 error message
 differenc, refs #1482

---
 tests/test_canned_queries.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index 4186a97c..b8c2baec 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -353,4 +353,4 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c
         "/data.json?sql=select+:_header_host&_shape=array"
     )
     assert 400 == response.status
-    assert "You did not supply a value for binding 1." == response.json["error"]
+    assert response.json["error"].startswith("You did not supply a value for binding")

From 875117c343e454221ef023be6ad977fdaea3ceda Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 9 Oct 2021 18:14:56 -0700
Subject: [PATCH 0459/1866] Fix bug with ?_next=x&_sort=rowid, closes #1470

---
 datasette/views/table.py | 6 ++++--
 tests/test_html.py       | 6 ++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 83f7c7cb..e8c66d3c 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -588,13 +588,15 @@ class TableView(RowTableShared):
         _next = _next or special_args.get("_next")
         offset = ""
         if _next:
+            sort_value = None
             if is_view:
                 # _next is an offset
                 offset = f" offset {int(_next)}"
             else:
                 components = urlsafe_components(_next)
-                # If a sort order is applied, the first of these is the sort value
-                if sort or sort_desc:
+                # If a sort order is applied and there are multiple components,
+                # the first of these is the sort value
+                if (sort or sort_desc) and (len(components) > 1):
                     sort_value = components[0]
                     # Special case for if non-urlencoded first token was $null
                     if _next.split(",")[0] == "$null":
diff --git a/tests/test_html.py b/tests/test_html.py
index e73ccd2f..c22d87da 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -1817,3 +1817,9 @@ def test_facet_total_shown_if_facet_max_size(use_facet_size_max):
             assert fragment in response.text
         else:
             assert fragment not in response.text
+
+
+def test_sort_rowid_with_next(app_client):
+    # https://github.com/simonw/datasette/issues/1470
+    response = app_client.get("/fixtures/binary_data?_size=1&_next=1&_sort=rowid")
+    assert response.status == 200

From 0d5cc20aeffa3537cfc9296d01ec24b9c6e23dcf Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 9 Oct 2021 18:25:33 -0700
Subject: [PATCH 0460/1866] Revert "asyncio_run helper to deal with a 3.10
 warning, refs #1482"

This reverts commit 98dcabccbbf9c0800efa74df9b7d1fee81c3cd0c.
---
 datasette/cli.py            | 13 ++++++++-----
 datasette/utils/__init__.py |  9 ---------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/datasette/cli.py b/datasette/cli.py
index 95ee9495..22e2338a 100644
--- a/datasette/cli.py
+++ b/datasette/cli.py
@@ -14,7 +14,6 @@ from runpy import run_module
 import webbrowser
 from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
 from .utils import (
-    asyncio_run,
     StartupError,
     check_connection,
     find_spatialite,
@@ -137,7 +136,9 @@ def cli():
 @click.option("--inspect-file", default="-")
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
-    inspect_data = asyncio_run(inspect_(files, sqlite_extensions))
+    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
+    loop = asyncio.get_event_loop()
+    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
     if inspect_file == "-":
         sys.stdout.write(json.dumps(inspect_data, indent=2))
     else:
@@ -554,10 +555,10 @@ def serve(
         return ds
 
     # Run the "startup" plugin hooks
-    asyncio_run(ds.invoke_startup())
+    asyncio.get_event_loop().run_until_complete(ds.invoke_startup())
 
     # Run async soundness checks - but only if we're not under pytest
-    asyncio_run(check_databases(ds))
+    asyncio.get_event_loop().run_until_complete(check_databases(ds))
 
     if get:
         client = TestClient(ds)
@@ -577,7 +578,9 @@ def serve(
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage
-            path = asyncio_run(initial_path_for_datasette(ds))
+            path = asyncio.get_event_loop().run_until_complete(
+                initial_path_for_datasette(ds)
+            )
             url = f"http://{host}:{port}{path}"
         webbrowser.open(url)
     uvicorn_kwargs = dict(
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 1b7bfe05..70ac8976 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,12 +1089,3 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
-
-
-def asyncio_run(coro):
-    if hasattr(asyncio, "run"):
-        # Added in Python 3.7
-        return asyncio.run(coro)
-    else:
-        loop = asyncio.get_event_loop()
-        return loop.run_until_complete(coro)

From b50bf5d13fed10d0b930f62198d8f2658e15e1eb Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 13 Oct 2021 14:08:06 -0700
Subject: [PATCH 0461/1866] Don't persist _next in hidden field, closes #1483

---
 datasette/views/table.py |  2 +-
 tests/test_html.py       | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index e8c66d3c..efcef4d2 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -889,7 +889,7 @@ class TableView(RowTableShared):
 
             form_hidden_args = []
             for key in request.args:
-                if key.startswith("_") and key not in ("_sort", "_search"):
+                if key.startswith("_") and key not in ("_sort", "_search", "_next"):
                     for value in request.args.getlist(key):
                         form_hidden_args.append((key, value))
 
diff --git a/tests/test_html.py b/tests/test_html.py
index c22d87da..151ac5c3 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -563,6 +563,16 @@ def test_facets_persist_through_filter_form(app_client):
     ]
 
 
+def test_next_does_not_persist_in_hidden_field(app_client):
+    response = app_client.get("/fixtures/searchable?_size=1&_next=1")
+    assert response.status == 200
+    inputs = Soup(response.body, "html.parser").find("form").findAll("input")
+    hiddens = [i for i in inputs if i["type"] == "hidden"]
+    assert [(hidden["name"], hidden["value"]) for hidden in hiddens] == [
+        ("_size", "1"),
+    ]
+
+
 @pytest.mark.parametrize(
     "path,expected_classes",
     [

From 68087440b3448633a62807c1623559619584f2ee Mon Sep 17 00:00:00 2001
From: Rhet Turnbull 
Date: Wed, 13 Oct 2021 14:09:10 -0700
Subject: [PATCH 0462/1866] Added instructions for installing plugins via pipx

Closes #1486
---
 docs/installation.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/installation.rst b/docs/installation.rst
index b6881bc0..723f1e3f 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -90,6 +90,25 @@ Once pipx is installed you can use it to install Datasette like this::
 
 Then run ``datasette --version`` to confirm that it has been successfully installed.
 
+Installing plugins using pipx
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can install additional datasette plugins with ``pipx inject`` like so::
+
+    $ pipx inject datasette datasette-json-html
+    injected package datasette-json-html into venv datasette
+    done! ✨ 🌟 ✨
+
+    $ datasette plugins
+    [
+        {
+            "name": "datasette-json-html",
+            "static": false,
+            "templates": false,
+            "version": "0.6"
+        }
+    ]
+
 Upgrading packages using pipx
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 759fd97a54638c1a5e2cac65bac0ac7c07ce2305 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:09:23 -0700
Subject: [PATCH 0463/1866] Update pytest-timeout requirement from <1.5,>=1.4.2
 to >=1.4.2,<2.1 (#1485)

Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-timeout/releases)
- [Commits](https://github.com/pytest-dev/pytest-timeout/commits)

---
updated-dependencies:
- dependency-name: pytest-timeout
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 84f32087..da62276e 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,7 @@ setup(
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.7b0",
-            "pytest-timeout>=1.4.2,<1.5",
+            "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],
         "rich": ["rich"],

From 6aab0217f07bff4556cc92885a14279d5b295f84 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:10:03 -0700
Subject: [PATCH 0464/1866] Update pytest-xdist requirement from <2.4,>=2.2.1
 to >=2.2.1,<2.5 (#1476)

Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-xdist/releases)
- [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.2.1...v2.4.0)

---
updated-dependencies:
- dependency-name: pytest-xdist
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index da62276e..571bb35a 100644
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ setup(
         "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell"],
         "test": [
             "pytest>=5.2.2,<6.3.0",
-            "pytest-xdist>=2.2.1,<2.4",
+            "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
             "black==21.7b0",

From 31352914c427162f785d2610222a54a426d5215f Mon Sep 17 00:00:00 2001
From: Michael Tiemann <72577720+MichaelTiemannOSC@users.noreply.github.com>
Date: Wed, 13 Oct 2021 17:10:23 -0400
Subject: [PATCH 0465/1866] Update full_text_search.rst (#1474)

Change "above" to "below" to correct correspondence of reference to example.
---
 docs/full_text_search.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst
index 90b2e8c1..a285417f 100644
--- a/docs/full_text_search.rst
+++ b/docs/full_text_search.rst
@@ -47,7 +47,7 @@ If that option has been specified in the table metadata but you want to over-rid
 Configuring full-text search for a table or view
 ------------------------------------------------
 
-If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown above, Datasette will detect it automatically and add a search interface to the table page for that table.
+If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
 You can also manually configure which table should be used for full-text search using query string parameters or :ref:`metadata`. You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 

From a673a93b57e249f06b2d0265ce33f458258feeb0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:11:00 -0700
Subject: [PATCH 0466/1866] Update pluggy requirement from ~=0.13.0 to
 >=0.13,<1.1 (#1448)

Updates the requirements on [pluggy](https://github.com/pytest-dev/pluggy) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pluggy/releases)
- [Changelog](https://github.com/pytest-dev/pluggy/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pluggy/compare/0.13.0...1.0.0)

---
updated-dependencies:
- dependency-name: pluggy
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 571bb35a..cb1cec66 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ setup(
         "hupper~=1.9",
         "httpx>=0.17",
         "pint~=0.9",
-        "pluggy~=0.13.0",
+        "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.8",
         "janus>=0.4,<0.7",

From 763d0a0faabc6b53fa21ea2f0914e83ca12dfb34 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Wed, 13 Oct 2021 14:19:53 -0700
Subject: [PATCH 0467/1866] Fix for cog menu default facet bug, closes #1469

---
 datasette/static/table.js      | 9 ++++-----
 datasette/templates/table.html | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/datasette/static/table.js b/datasette/static/table.js
index 85bf073f..3c88cc40 100644
--- a/datasette/static/table.js
+++ b/datasette/static/table.js
@@ -129,16 +129,15 @@ var DROPDOWN_ICON_SVG = ` el.dataset.column);
     var isFirstColumn =
       th.parentElement.querySelector("th:first-of-type") == th;
     var isSinglePk =
       th.getAttribute("data-is-pk") == "1" &&
       document.querySelectorAll('th[data-is-pk="1"]').length == 1;
-    if (
-      isFirstColumn ||
-      params.getAll("_facet").includes(column) ||
-      isSinglePk
-    ) {
+    if (isFirstColumn || displayedFacets.includes(column) || isSinglePk) {
       facetItem.parentNode.style.display = "none";
     } else {
       facetItem.parentNode.style.display = "block";
diff --git a/datasette/templates/table.html b/datasette/templates/table.html
index 6ba301b5..4b9df8e1 100644
--- a/datasette/templates/table.html
+++ b/datasette/templates/table.html
@@ -154,7 +154,7 @@
 {% if facet_results %}
     
    
         {% for facet_info in sorted_facet_results %}
-            
    
+            
    
                 
    
                     {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
                         {% if show_facet_counts %} {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}{% endif %}

From e1012e7098056734d9c90f081493991009253390 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 14:47:42 -0700
Subject: [PATCH 0468/1866] Bump black from 21.7b0 to 21.9b0 (#1471)

Bumps [black](https://github.com/psf/black) from 21.7b0 to 21.9b0.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/commits)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index cb1cec66..20576e72 100644
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,7 @@ setup(
             "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
             "beautifulsoup4>=4.8.1,<4.10.0",
-            "black==21.7b0",
+            "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",
         ],

From 2a8c6690399ee832ee62aafdede1794f5945d911 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 13 Oct 2021 15:35:36 -0700
Subject: [PATCH 0469/1866] Update beautifulsoup4 requirement (#1463)

Updates the requirements on [beautifulsoup4](http://www.crummy.com/software/BeautifulSoup/bs4/) to permit the latest version.

---
updated-dependencies:
- dependency-name: beautifulsoup4
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 20576e72..905fed16 100644
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ setup(
             "pytest>=5.2.2,<6.3.0",
             "pytest-xdist>=2.2.1,<2.5",
             "pytest-asyncio>=0.10,<0.16",
-            "beautifulsoup4>=4.8.1,<4.10.0",
+            "beautifulsoup4>=4.8.1,<4.11.0",
             "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",
             "trustme>=0.7,<0.10",

From b267b5775436577a91a9f9655143908aecff05da Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 11:03:44 -0700
Subject: [PATCH 0470/1866] Upgrade to httpx 0.20

* Upgrade to httpx 0.20, closes #1488
* TestClient.post() should not default to following redirects
---
 datasette/app.py                         |  5 +++-
 datasette/utils/asgi.py                  |  4 +--
 datasette/utils/testing.py               | 20 ++++++-------
 setup.py                                 |  2 +-
 tests/test_api.py                        | 22 ++++++--------
 tests/test_auth.py                       |  5 +---
 tests/test_canned_queries.py             | 12 +-------
 tests/test_custom_pages.py               |  4 +--
 tests/test_html.py                       | 38 ++++++++++++------------
 tests/test_internals_datasette_client.py |  1 -
 tests/test_internals_request.py          |  5 +++-
 tests/test_plugins.py                    | 13 ++++----
 12 files changed, 60 insertions(+), 71 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 06db740e..1f69c2b3 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -1139,6 +1139,7 @@ class DatasetteRouter:
         raw_path = scope.get("raw_path")
         if raw_path:
             path = raw_path.decode("ascii")
+        path = path.partition("?")[0]
         return await self.route_path(scope, receive, send, path)
 
     async def route_path(self, scope, receive, send, path):
@@ -1192,7 +1193,9 @@ class DatasetteRouter:
 
     async def handle_404(self, request, send, exception=None):
         # If URL has a trailing slash, redirect to URL without it
-        path = request.scope.get("raw_path", request.scope["path"].encode("utf8"))
+        path = request.scope.get(
+            "raw_path", request.scope["path"].encode("utf8")
+        ).partition(b"?")[0]
         context = {}
         if path.endswith(b"/"):
             path = path.rstrip(b"/")
diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 5fa03b0a..696944df 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -75,7 +75,7 @@ class Request:
     @property
     def path(self):
         if self.scope.get("raw_path") is not None:
-            return self.scope["raw_path"].decode("latin-1")
+            return self.scope["raw_path"].decode("latin-1").partition("?")[0]
         else:
             path = self.scope["path"]
             if isinstance(path, str):
@@ -122,7 +122,7 @@ class Request:
             "http_version": "1.1",
             "method": method,
             "path": path,
-            "raw_path": path.encode("latin-1"),
+            "raw_path": path_with_query_string.encode("latin-1"),
             "query_string": query_string.encode("latin-1"),
             "scheme": scheme,
             "type": "http",
diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py
index a169a83d..94750b1f 100644
--- a/datasette/utils/testing.py
+++ b/datasette/utils/testing.py
@@ -55,10 +55,10 @@ class TestClient:
 
     @async_to_sync
     async def get(
-        self, path, allow_redirects=True, redirect_count=0, method="GET", cookies=None
+        self, path, follow_redirects=False, redirect_count=0, method="GET", cookies=None
     ):
         return await self._request(
-            path, allow_redirects, redirect_count, method, cookies
+            path, follow_redirects, redirect_count, method, cookies
         )
 
     @async_to_sync
@@ -67,7 +67,7 @@ class TestClient:
         path,
         post_data=None,
         body=None,
-        allow_redirects=True,
+        follow_redirects=False,
         redirect_count=0,
         content_type="application/x-www-form-urlencoded",
         cookies=None,
@@ -90,7 +90,7 @@ class TestClient:
             body = urlencode(post_data, doseq=True)
         return await self._request(
             path=path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             redirect_count=redirect_count,
             method="POST",
             cookies=cookies,
@@ -103,7 +103,7 @@ class TestClient:
     async def request(
         self,
         path,
-        allow_redirects=True,
+        follow_redirects=True,
         redirect_count=0,
         method="GET",
         cookies=None,
@@ -113,7 +113,7 @@ class TestClient:
     ):
         return await self._request(
             path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             redirect_count=redirect_count,
             method=method,
             cookies=cookies,
@@ -125,7 +125,7 @@ class TestClient:
     async def _request(
         self,
         path,
-        allow_redirects=True,
+        follow_redirects=True,
         redirect_count=0,
         method="GET",
         cookies=None,
@@ -139,19 +139,19 @@ class TestClient:
         httpx_response = await self.ds.client.request(
             method,
             path,
-            allow_redirects=allow_redirects,
+            follow_redirects=follow_redirects,
             avoid_path_rewrites=True,
             cookies=cookies,
             headers=headers,
             content=post_body,
         )
         response = TestResponse(httpx_response)
-        if allow_redirects and response.status in (301, 302):
+        if follow_redirects and response.status in (301, 302):
             assert (
                 redirect_count < self.max_redirects
             ), f"Redirected {redirect_count} times, max_redirects={self.max_redirects}"
             location = response.headers["Location"]
             return await self._request(
-                location, allow_redirects=True, redirect_count=redirect_count + 1
+                location, follow_redirects=True, redirect_count=redirect_count + 1
             )
         return response
diff --git a/setup.py b/setup.py
index 905fed16..45eac040 100644
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@ setup(
         "click-default-group~=1.2.2",
         "Jinja2>=2.10.3,<3.1.0",
         "hupper~=1.9",
-        "httpx>=0.17",
+        "httpx>=0.20",
         "pint~=0.9",
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
diff --git a/tests/test_api.py b/tests/test_api.py
index 1e93c62e..38d1ba08 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -629,7 +629,7 @@ def test_no_files_uses_memory_database(app_client_no_files):
     ),
 )
 def test_old_memory_urls_redirect(app_client_no_files, path, expected_redirect):
-    response = app_client_no_files.get(path, allow_redirects=False)
+    response = app_client_no_files.get(path)
     assert response.status == 301
     assert response.headers["location"] == expected_redirect
 
@@ -708,12 +708,8 @@ def test_table_not_exists_json(app_client):
 
 
 def test_jsono_redirects_to_shape_objects(app_client_with_hash):
-    response_1 = app_client_with_hash.get(
-        "/fixtures/simple_primary_key.jsono", allow_redirects=False
-    )
-    response = app_client_with_hash.get(
-        response_1.headers["Location"], allow_redirects=False
-    )
+    response_1 = app_client_with_hash.get("/fixtures/simple_primary_key.jsono")
+    response = app_client_with_hash.get(response_1.headers["Location"])
     assert response.status == 302
     assert response.headers["Location"].endswith("?_shape=objects")
 
@@ -1488,7 +1484,7 @@ def test_settings_json(app_client):
     ),
 )
 def test_config_redirects_to_settings(app_client, path, expected_redirect):
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 301
     assert response.headers["Location"] == expected_redirect
 
@@ -1834,9 +1830,7 @@ def test_hash_parameter(
     current_hash = app_client_two_attached_databases_one_immutable.ds.databases[
         "fixtures"
     ].hash[:7]
-    response = app_client_two_attached_databases_one_immutable.get(
-        path, allow_redirects=False
-    )
+    response = app_client_two_attached_databases_one_immutable.get(path)
     assert response.status == 302
     location = response.headers["Location"]
     assert expected_redirect.replace("HASH", current_hash) == location
@@ -1844,7 +1838,7 @@ def test_hash_parameter(
 
 def test_hash_parameter_ignored_for_mutable_databases(app_client):
     path = "/fixtures/facetable.json?_hash=1"
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 200
 
 
@@ -1976,7 +1970,9 @@ def test_cors(app_client_with_cors, path, status_code):
     ),
 )
 def test_database_with_space_in_name(app_client_two_attached_databases, path):
-    response = app_client_two_attached_databases.get("/extra database" + path)
+    response = app_client_two_attached_databases.get(
+        "/extra database" + path, follow_redirects=True
+    )
     assert response.status == 200
 
 
diff --git a/tests/test_auth.py b/tests/test_auth.py
index 16397b7a..974f89ea 100644
--- a/tests/test_auth.py
+++ b/tests/test_auth.py
@@ -10,7 +10,6 @@ def test_auth_token(app_client):
     path = f"/-/auth-token?token={app_client.ds._root_token}"
     response = app_client.get(
         path,
-        allow_redirects=False,
     )
     assert 302 == response.status
     assert "/" == response.headers["Location"]
@@ -23,7 +22,6 @@ def test_auth_token(app_client):
         403
         == app_client.get(
             path,
-            allow_redirects=False,
         ).status
     )
 
@@ -78,14 +76,13 @@ def test_logout(app_client):
         in response2.text
     )
     # If logged out you get a redirect to /
-    response3 = app_client.get("/-/logout", allow_redirects=False)
+    response3 = app_client.get("/-/logout")
     assert 302 == response3.status
     # A POST to that page should log the user out
     response4 = app_client.post(
         "/-/logout",
         csrftoken_from=True,
         cookies={"ds_actor": app_client.actor_cookie({"id": "test"})},
-        allow_redirects=False,
     )
     # The ds_actor cookie should have been unset
     assert response4.cookie_was_deleted("ds_actor")
diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py
index b8c2baec..cea81ec7 100644
--- a/tests/test_canned_queries.py
+++ b/tests/test_canned_queries.py
@@ -59,7 +59,6 @@ def test_insert(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         {"name": "Hello"},
-        allow_redirects=False,
         csrftoken_from=True,
         cookies={"foo": "bar"},
     )
@@ -95,16 +94,13 @@ def test_insert_with_cookies_requires_csrf(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         {"name": "Hello"},
-        allow_redirects=False,
         cookies={"foo": "bar"},
     )
     assert 403 == response.status
 
 
 def test_insert_no_cookies_no_csrf(canned_write_client):
-    response = canned_write_client.post(
-        "/data/add_name", {"name": "Hello"}, allow_redirects=False
-    )
+    response = canned_write_client.post("/data/add_name", {"name": "Hello"})
     assert 302 == response.status
     assert "/data/add_name?success" == response.headers["Location"]
 
@@ -114,7 +110,6 @@ def test_custom_success_message(canned_write_client):
         "/data/delete_name",
         {"rowid": 1},
         cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert 302 == response.status
@@ -129,7 +124,6 @@ def test_insert_error(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name_specify_id",
         {"rowid": 1, "name": "Should fail"},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert 302 == response.status
@@ -145,7 +139,6 @@ def test_insert_error(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name_specify_id",
         {"rowid": 1, "name": "Should fail"},
-        allow_redirects=False,
         csrftoken_from=True,
     )
     assert [["ERROR", 3]] == canned_write_client.ds.unsign(
@@ -168,7 +161,6 @@ def test_json_post_body(canned_write_client):
     response = canned_write_client.post(
         "/data/add_name",
         body=json.dumps({"name": ["Hello", "there"]}),
-        allow_redirects=False,
     )
     assert 302 == response.status
     assert "/data/add_name?success" == response.headers["Location"]
@@ -189,7 +181,6 @@ def test_json_response(canned_write_client, headers, body, querystring):
     response = canned_write_client.post(
         "/data/add_name" + (querystring or ""),
         body=body,
-        allow_redirects=False,
         headers=headers,
     )
     assert 200 == response.status
@@ -331,7 +322,6 @@ def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_js
         f"/data/runme_post{qs}",
         {},
         csrftoken_from=use_csrf or None,
-        allow_redirects=False,
     )
     if return_json:
         assert response.status == 200
diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py
index 76c67397..66b7437a 100644
--- a/tests/test_custom_pages.py
+++ b/tests/test_custom_pages.py
@@ -67,13 +67,13 @@ def test_custom_content_type(custom_pages_client):
 
 
 def test_redirect(custom_pages_client):
-    response = custom_pages_client.get("/redirect", allow_redirects=False)
+    response = custom_pages_client.get("/redirect")
     assert 302 == response.status
     assert "/example" == response.headers["Location"]
 
 
 def test_redirect2(custom_pages_client):
-    response = custom_pages_client.get("/redirect2", allow_redirects=False)
+    response = custom_pages_client.get("/redirect2")
     assert 301 == response.status
     assert "/example" == response.headers["Location"]
 
diff --git a/tests/test_html.py b/tests/test_html.py
index 151ac5c3..5f2ba2f1 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -100,9 +100,9 @@ def test_not_allowed_methods():
 
 
 def test_database_page_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get("/fixtures", allow_redirects=False)
-    assert response.status == 302
     response = app_client_with_hash.get("/fixtures")
+    assert response.status == 302
+    response = app_client_with_hash.get("/fixtures", follow_redirects=True)
     assert "fixtures" in response.text
 
 
@@ -161,22 +161,22 @@ def test_sql_time_limit(app_client_shorter_time_limit):
 
 
 def test_row_redirects_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get(
-        "/fixtures/simple_primary_key/1", allow_redirects=False
-    )
+    response = app_client_with_hash.get("/fixtures/simple_primary_key/1")
     assert response.status == 302
     assert response.headers["Location"].endswith("/1")
-    response = app_client_with_hash.get("/fixtures/simple_primary_key/1")
+    response = app_client_with_hash.get(
+        "/fixtures/simple_primary_key/1", follow_redirects=True
+    )
     assert response.status == 200
 
 
 def test_row_strange_table_name_with_url_hash(app_client_with_hash):
-    response = app_client_with_hash.get(
-        "/fixtures/table%2Fwith%2Fslashes.csv/3", allow_redirects=False
-    )
+    response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3")
     assert response.status == 302
     assert response.headers["Location"].endswith("/table%2Fwith%2Fslashes.csv/3")
-    response = app_client_with_hash.get("/fixtures/table%2Fwith%2Fslashes.csv/3")
+    response = app_client_with_hash.get(
+        "/fixtures/table%2Fwith%2Fslashes.csv/3", follow_redirects=True
+    )
     assert response.status == 200
 
 
@@ -255,13 +255,13 @@ def test_add_filter_redirects(app_client):
     )
     path_base = "/fixtures/simple_primary_key"
     path = path_base + "?" + filter_args
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?content__startswith=x")
 
     # Adding a redirect to an existing query string:
     path = path_base + "?foo=bar&" + filter_args
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?foo=bar&content__startswith=x")
 
@@ -277,7 +277,7 @@ def test_add_filter_redirects(app_client):
             }
         )
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?content__isnull=5")
 
@@ -299,7 +299,7 @@ def test_existing_filter_redirects(app_client):
     }
     path_base = "/fixtures/simple_primary_key"
     path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert_querystring_equal(
         "name__contains=hello&age__gte=22&age__lt=30&name__contains=world",
@@ -309,7 +309,7 @@ def test_existing_filter_redirects(app_client):
     # Setting _filter_column_3 to empty string should remove *_3 entirely
     filter_args["_filter_column_3"] = ""
     path = path_base + "?" + urllib.parse.urlencode(filter_args)
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert_querystring_equal(
         "name__contains=hello&age__gte=22&name__contains=world",
@@ -317,7 +317,7 @@ def test_existing_filter_redirects(app_client):
     )
 
     # ?_filter_op=exact should be removed if unaccompanied by _fiter_column
-    response = app_client.get(path_base + "?_filter_op=exact", allow_redirects=False)
+    response = app_client.get(path_base + "?_filter_op=exact")
     assert response.status == 302
     assert "?" not in response.headers["Location"]
 
@@ -336,7 +336,7 @@ def test_empty_search_parameter_gets_removed(app_client):
             }
         )
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?name__exact=chidi")
 
@@ -360,7 +360,7 @@ def test_sort_by_desc_redirects(app_client):
         + "?"
         + urllib.parse.urlencode({"_sort": "sortable", "_sort_by_desc": "1"})
     )
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert response.status == 302
     assert response.headers["Location"].endswith("?_sort_desc=sortable")
 
@@ -1148,7 +1148,7 @@ def test_404(app_client, path):
     [("/fixtures/", "/fixtures"), ("/fixtures/simple_view/", "/fixtures/simple_view")],
 )
 def test_404_trailing_slash_redirect(app_client, path, expected_redirect):
-    response = app_client.get(path, allow_redirects=False)
+    response = app_client.get(path)
     assert 302 == response.status
     assert expected_redirect == response.headers["Location"]
 
diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py
index c538bef1..8c5b5bd3 100644
--- a/tests/test_internals_datasette_client.py
+++ b/tests/test_internals_datasette_client.py
@@ -42,7 +42,6 @@ async def test_client_post(datasette, prefix):
             data={
                 "message": "A message",
             },
-            allow_redirects=False,
         )
         assert isinstance(response, httpx.Response)
         assert response.status_code == 302
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index fe273645..c42cfbd3 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -97,11 +97,14 @@ def test_request_url_vars():
     [("/", "", "/"), ("/", "foo=bar", "/?foo=bar"), ("/foo", "bar", "/foo?bar")],
 )
 def test_request_properties(path, query_string, expected_full_path):
+    path_with_query_string = path
+    if query_string:
+        path_with_query_string += "?" + query_string
     scope = {
         "http_version": "1.1",
         "method": "POST",
         "path": path,
-        "raw_path": path.encode("latin-1"),
+        "raw_path": path_with_query_string.encode("latin-1"),
         "query_string": query_string.encode("latin-1"),
         "scheme": "http",
         "type": "http",
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index a024c39b..7dac8002 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -71,7 +71,7 @@ def test_hook_plugin_prepare_connection_arguments(app_client):
             },
         ),
         (
-            "/fixtures/",
+            "/fixtures",
             {
                 "template": "database.html",
                 "database": "fixtures",
@@ -106,6 +106,7 @@ def test_hook_plugin_prepare_connection_arguments(app_client):
 )
 def test_hook_extra_css_urls(app_client, path, expected_decoded_object):
     response = app_client.get(path)
+    assert response.status == 200
     links = Soup(response.body, "html.parser").findAll("link")
     special_href = [
         l for l in links if l.attrs["href"].endswith("/extra-css-urls-demo.css")
@@ -263,7 +264,7 @@ def test_plugin_config_file(app_client):
             },
         ),
         (
-            "/fixtures/",
+            "/fixtures",
             {
                 "template": "database.html",
                 "database": "fixtures",
@@ -640,7 +641,7 @@ async def test_hook_permission_allowed(app_client, action, expected):
 def test_actor_json(app_client):
     assert {"actor": None} == app_client.get("/-/actor.json").json
     assert {"actor": {"id": "bot2", "1+1": 2}} == app_client.get(
-        "/-/actor.json/?_bot2=1"
+        "/-/actor.json?_bot2=1"
     ).json
 
 
@@ -674,7 +675,7 @@ def test_hook_register_routes_with_datasette(configured_path):
         assert configured_path.upper() == response.text
         # Other one should 404
         other_path = [p for p in ("path1", "path2") if configured_path != p][0]
-        assert client.get(f"/{other_path}/").status == 404
+        assert client.get(f"/{other_path}/", follow_redirects=True).status == 404
 
 
 def test_hook_register_routes_post(app_client):
@@ -777,7 +778,7 @@ def test_hook_register_magic_parameters(restore_working_directory):
         },
     ) as client:
         response = client.post("/data/runme", {}, csrftoken_from=True)
-        assert 200 == response.status
+        assert 302 == response.status
         actual = client.get("/data/logs.json?_sort_desc=rowid&_shape=array").json
         assert [{"rowid": 1, "line": "1.1"}] == actual
         # Now try the GET request against get_uuid
@@ -794,7 +795,7 @@ def test_hook_forbidden(restore_working_directory):
     ) as client:
         response = client.get("/")
         assert 403 == response.status
-        response2 = client.get("/data2", allow_redirects=False)
+        response2 = client.get("/data2")
         assert 302 == response2.status
         assert "/login?message=view-database" == response2.headers["Location"]
         assert "view-database" == client.ds._last_forbidden_message

From 827fa823d1b919c445f3141174ecb7a82717d99c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 14 Oct 2021 11:10:42 -0700
Subject: [PATCH 0471/1866] Update pyyaml requirement from ~=5.3 to >=5.3,<7.0
 (#1489)

Updates the requirements on [pyyaml](https://github.com/yaml/pyyaml) to permit the latest version.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/5.3...6.0)

---
updated-dependencies:
- dependency-name: pyyaml
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 45eac040..c1b87bd4 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ setup(
         "aiofiles>=0.4,<0.8",
         "janus>=0.4,<0.7",
         "asgi-csrf>=0.9",
-        "PyYAML~=5.3",
+        "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",
         "itsdangerous>=1.1,<3.0",
         "python-baseconv==1.2.2",

From 0fdbf004843850f200e077a3c87427fe16c18b85 Mon Sep 17 00:00:00 2001
From: "C. Titus Brown" 
Date: Thu, 14 Oct 2021 11:39:55 -0700
Subject: [PATCH 0472/1866] Rework the `--static` documentation

Rework the `--static` documentation to better differentiate between the filesystem and serving locations. Closes #1457

Co-authored-by: Simon Willison 
---
 docs/custom_templates.rst | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst
index efb5b842..3e4eb633 100644
--- a/docs/custom_templates.rst
+++ b/docs/custom_templates.rst
@@ -173,18 +173,18 @@ Datasette can serve static files for you, using the ``--static`` option.
 Consider the following directory structure::
 
     metadata.json
-    static/styles.css
-    static/app.js
+    static-files/styles.css
+    static-files/app.js
 
-You can start Datasette using ``--static static:static/`` to serve those
-files from the ``/static/`` mount point::
+You can start Datasette using ``--static assets:static-files/`` to serve those
+files from the ``/assets/`` mount point::
 
-    $ datasette -m metadata.json --static static:static/ --memory
+    $ datasette -m metadata.json --static assets:static-files/ --memory
 
 The following URLs will now serve the content from those CSS and JS files::
 
-    http://localhost:8001/static/styles.css
-    http://localhost:8001/static/app.js
+    http://localhost:8001/assets/styles.css
+    http://localhost:8001/assets/app.js
 
 You can reference those files from ``metadata.json`` like so:
 
@@ -192,10 +192,10 @@ You can reference those files from ``metadata.json`` like so:
 
     {
         "extra_css_urls": [
-            "/static/styles.css"
+            "/assets/styles.css"
         ],
         "extra_js_urls": [
-            "/static/app.js"
+            "/assets/app.js"
         ]
     }
 
@@ -205,10 +205,10 @@ Publishing static assets
 The :ref:`cli_publish` command can be used to publish your static assets,
 using the same syntax as above::
 
-    $ datasette publish cloudrun mydb.db --static static:static/
+    $ datasette publish cloudrun mydb.db --static assets:static-files/
 
-This will upload the contents of the ``static/`` directory as part of the
-deployment, and configure Datasette to correctly serve the assets.
+This will upload the contents of the ``static-files/`` directory as part of the
+deployment, and configure Datasette to correctly serve the assets from ``/assets/``.
 
 .. _customization_custom_templates:
 

From 85849935292e500ab7a99f8fe0f9546e903baad3 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:03:28 -0700
Subject: [PATCH 0473/1866] --cors Access-Control-Allow-Headers: Authorization

Refs #1467, refs https://github.com/simonw/datasette-auth-tokens/issues/4
---
 datasette/app.py            | 3 ++-
 datasette/utils/__init__.py | 5 +++++
 datasette/views/base.py     | 9 +++++----
 datasette/views/database.py | 3 ++-
 datasette/views/index.py    | 4 ++--
 datasette/views/special.py  | 4 ++--
 tests/test_api.py           | 3 ++-
 7 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/datasette/app.py b/datasette/app.py
index 1f69c2b3..52c5e629 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -46,6 +46,7 @@ from .database import Database, QueryInterrupted
 from .utils import (
     PrefixedUrlString,
     StartupError,
+    add_cors_headers,
     async_call_with_supported_arguments,
     await_me_maybe,
     call_with_supported_arguments,
@@ -1321,7 +1322,7 @@ class DatasetteRouter:
         )
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         if request.path.split("?")[0].endswith(".json"):
             await asgi_send_json(send, info, status=status, headers=headers)
         else:
diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py
index 70ac8976..c339113c 100644
--- a/datasette/utils/__init__.py
+++ b/datasette/utils/__init__.py
@@ -1089,3 +1089,8 @@ async def derive_named_parameters(db, sql):
         return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"]
     except sqlite3.DatabaseError:
         return possible_params
+
+
+def add_cors_headers(headers):
+    headers["Access-Control-Allow-Origin"] = "*"
+    headers["Access-Control-Allow-Headers"] = "Authorization"
diff --git a/datasette/views/base.py b/datasette/views/base.py
index 3333781c..01e90220 100644
--- a/datasette/views/base.py
+++ b/datasette/views/base.py
@@ -11,6 +11,7 @@ import pint
 from datasette import __version__
 from datasette.database import QueryInterrupted
 from datasette.utils import (
+    add_cors_headers,
     await_me_maybe,
     EscapeHtmlWriter,
     InvalidSql,
@@ -163,7 +164,7 @@ class DataView(BaseView):
     async def options(self, request, *args, **kwargs):
         r = Response.text("ok")
         if self.ds.cors:
-            r.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(r.headers)
         return r
 
     def redirect(self, request, path, forward_querystring=True, remove_args=None):
@@ -174,7 +175,7 @@ class DataView(BaseView):
         r = Response.redirect(path)
         r.headers["Link"] = f"<{path}>; rel=preload"
         if self.ds.cors:
-            r.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(r.headers)
         return r
 
     async def data(self, request, database, hash, **kwargs):
@@ -417,7 +418,7 @@ class DataView(BaseView):
 
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         if request.args.get("_dl", None):
             if not trace:
                 content_type = "text/csv; charset=utf-8"
@@ -643,5 +644,5 @@ class DataView(BaseView):
             response.headers["Cache-Control"] = ttl_header
         response.headers["Referrer-Policy"] = "no-referrer"
         if self.ds.cors:
-            response.headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(response.headers)
         return response
diff --git a/datasette/views/database.py b/datasette/views/database.py
index e3070ce6..affded9b 100644
--- a/datasette/views/database.py
+++ b/datasette/views/database.py
@@ -8,6 +8,7 @@ from urllib.parse import parse_qsl, urlencode
 import markupsafe
 
 from datasette.utils import (
+    add_cors_headers,
     await_me_maybe,
     check_visibility,
     derive_named_parameters,
@@ -176,7 +177,7 @@ class DatabaseDownload(DataView):
         filepath = db.path
         headers = {}
         if self.ds.cors:
-            headers["Access-Control-Allow-Origin"] = "*"
+            add_cors_headers(headers)
         headers["Transfer-Encoding"] = "chunked"
         return AsgiFileDownload(
             filepath,
diff --git a/datasette/views/index.py b/datasette/views/index.py
index e37643f9..18454759 100644
--- a/datasette/views/index.py
+++ b/datasette/views/index.py
@@ -1,7 +1,7 @@
 import hashlib
 import json
 
-from datasette.utils import check_visibility, CustomJSONEncoder
+from datasette.utils import add_cors_headers, check_visibility, CustomJSONEncoder
 from datasette.utils.asgi import Response
 from datasette.version import __version__
 
@@ -129,7 +129,7 @@ class IndexView(BaseView):
         if as_format:
             headers = {}
             if self.ds.cors:
-                headers["Access-Control-Allow-Origin"] = "*"
+                add_cors_headers(headers)
             return Response(
                 json.dumps({db["name"]: db for db in databases}, cls=CustomJSONEncoder),
                 content_type="application/json; charset=utf-8",
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 9750dd06..3cb626a5 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -1,6 +1,6 @@
 import json
 from datasette.utils.asgi import Response, Forbidden
-from datasette.utils import actor_matches_allow
+from datasette.utils import actor_matches_allow, add_cors_headers
 from .base import BaseView
 import secrets
 
@@ -23,7 +23,7 @@ class JsonDataView(BaseView):
         if as_format:
             headers = {}
             if self.ds.cors:
-                headers["Access-Control-Allow-Origin"] = "*"
+                add_cors_headers(headers)
             return Response(
                 json.dumps(data),
                 content_type="application/json; charset=utf-8",
diff --git a/tests/test_api.py b/tests/test_api.py
index 38d1ba08..311ae464 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -1955,7 +1955,8 @@ def test_trace(trace_debug):
 def test_cors(app_client_with_cors, path, status_code):
     response = app_client_with_cors.get(path)
     assert response.status == status_code
-    assert "*" == response.headers["Access-Control-Allow-Origin"]
+    assert response.headers["Access-Control-Allow-Origin"] == "*"
+    assert response.headers["Access-Control-Allow-Headers"] == "Authorization"
 
 
 @pytest.mark.parametrize(

From 81bf9a9f3cc5b30107a6b1adeee39d5e8312ecfc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:19:03 -0700
Subject: [PATCH 0474/1866] Updated --cors documentation, refs #1467

---
 docs/json_api.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/json_api.rst b/docs/json_api.rst
index 09cac1f9..7d3123b7 100644
--- a/docs/json_api.rst
+++ b/docs/json_api.rst
@@ -10,9 +10,10 @@ To access the API for a page, either click on the ``.json`` link on that page or
 edit the URL and add a ``.json`` extension to it.
 
 If you started Datasette with the ``--cors`` option, each JSON endpoint will be
-served with the following additional HTTP header::
+served with the following additional HTTP headers::
 
     Access-Control-Allow-Origin: *
+    Access-Control-Allow-Headers: Authorization
 
 This means JavaScript running on any domain will be able to make cross-origin
 requests to fetch the data.

From 8934507cdc0029a598cf37cdb3818fd31af5e33c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:22:19 -0700
Subject: [PATCH 0475/1866] Release 0.59

Refs #942, #1404, #1405, #1416, #1420, #1421, #1422, #1423, #1425, #1431, #1443, #1446, #1449, #1467, #1469, #1470, #1488
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 24 ++++++++----------------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/datasette/version.py b/datasette/version.py
index 87b18fab..d78b2e90 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59a2"
+__version__ = "0.59"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 737a151b..b2d95c49 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,36 +4,28 @@
 Changelog
 =========
 
-.. _v0_59a2:
+.. _v0_59:
 
-0.59a2 (2021-08-27)
--------------------
+0.59 (2021-10-14)
+-----------------
 
 - Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
 - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
+- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defalts to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
+- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 `__)
 - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
 - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)
 - ``datasette.databases`` property is now a documented API. (:issue:`1443`)
-- Datasette base template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
-
-.. _v0_59a1:
-
-0.59a1 (2021-08-08)
--------------------
-
+- The ``base.html`` template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
 - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`)
-
-.. _v0_59a0:
-
-0.59a0 (2021-08-06)
--------------------
-
 - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`)
 - New ``hide_sql`` canned query option for defaulting to hiding the SQL quey used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
 - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`)
 - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`)
 - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`)
+- Fixed bug where ``?_next=x&_sort=rowid`` could throw an error. (:issue:`1470`)
+- Column cog menu no longer shows the option to facet by a column that is already selected by the default facets in metadata. (:issue:`1469`)
 
 .. _v0_58_1:
 

From ff9ccfb0310501a3b4b4ca24d73246a8eb3e7914 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 14 Oct 2021 12:23:43 -0700
Subject: [PATCH 0476/1866] Fixed typo in release notes

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index b2d95c49..9a1edfe8 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -12,7 +12,7 @@ Changelog
 - Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`)
 - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
-- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defalts to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
+- Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
 - The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 `__)
 - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
 - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)

From e5d01ca5831844a02ef5b1ef0d5b9eb8bad3c9a4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 11:56:27 -0700
Subject: [PATCH 0477/1866] Fixed typo in release notes

---
 docs/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 9a1edfe8..24e19baa 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -20,7 +20,7 @@ Changelog
 - The ``base.html`` template now wraps everything other than the ``
    `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`)
 - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`)
 - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`)
-- New ``hide_sql`` canned query option for defaulting to hiding the SQL quey used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
+- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
 - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`)
 - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`)
 - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`)

From a1b20852db751acc445dce5f23e988d2c0299655 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 12:00:00 -0700
Subject: [PATCH 0478/1866] Unwrapped some documentation text

---
 docs/sql_queries.rst | 71 +++++++++++---------------------------------
 1 file changed, 17 insertions(+), 54 deletions(-)

diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 407e4ba2..4958b56a 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -3,34 +3,22 @@
 Running SQL queries
 ===================
 
-Datasette treats SQLite database files as read-only and immutable. This means it
-is not possible to execute INSERT or UPDATE statements using Datasette, which
-allows us to expose SELECT statements to the outside world without needing to
-worry about SQL injection attacks.
+Datasette treats SQLite database files as read-only and immutable. This means it is not possible to execute INSERT or UPDATE statements using Datasette, which allows us to expose SELECT statements to the outside world without needing to worry about SQL injection attacks.
 
-The easiest way to execute custom SQL against Datasette is through the web UI.
-The database index page includes a SQL editor that lets you run any SELECT query
-you like. You can also construct queries using the filter interface on the
-tables page, then click "View and edit SQL" to open that query in the custom
-SQL editor.
+The easiest way to execute custom SQL against Datasette is through the web UI. The database index page includes a SQL editor that lets you run any SELECT query you like. You can also construct queries using the filter interface on the tables page, then click "View and edit SQL" to open that query in the custom SQL editor.
 
-Note that this interface is only available if the :ref:`permissions_execute_sql`
-permission is allowed.
+Note that this interface is only available if the :ref:`permissions_execute_sql` permission is allowed.
 
-Any Datasette SQL query is reflected in the URL of the page, allowing you to
-bookmark them, share them with others and navigate through previous queries
-using your browser back button.
+Any Datasette SQL query is reflected in the URL of the page, allowing you to bookmark them, share them with others and navigate through previous queries using your browser back button.
 
-You can also retrieve the results of any query as JSON by adding ``.json`` to
-the base URL.
+You can also retrieve the results of any query as JSON by adding ``.json`` to the base URL.
 
 .. _sql_parameters:
 
 Named parameters
 ----------------
 
-Datasette has special support for SQLite named parameters. Consider a SQL query
-like this:
+Datasette has special support for SQLite named parameters. Consider a SQL query like this:
 
 .. code-block:: sql
 
@@ -38,17 +26,13 @@ like this:
     where "PermitNotes" like :notes
     and "qSpecies" = :species
 
-If you execute this query using the custom query editor, Datasette will extract
-the two named parameters and use them to construct form fields for you to
-provide values.
+If you execute this query using the custom query editor, Datasette will extract the two named parameters and use them to construct form fields for you to provide values.
 
 You can also provide values for these fields by constructing a URL::
 
     /mydatabase?sql=select...&species=44
 
-SQLite string escaping rules will be applied to values passed using named
-parameters - they will be wrapped in quotes and their content will be correctly
-escaped.
+SQLite string escaping rules will be applied to values passed using named parameters - they will be wrapped in quotes and their content will be correctly escaped.
 
 Datasette disallows custom SQL queries containing the string PRAGMA (with a small number `of exceptions `__) as SQLite pragma statements can be used to change database settings at runtime. If you need to include the string "pragma" in a query you can do so safely using a named parameter.
 
@@ -57,9 +41,7 @@ Datasette disallows custom SQL queries containing the string PRAGMA (with a smal
 Views
 -----
 
-If you want to bundle some pre-written SQL queries with your Datasette-hosted
-database you can do so in two ways. The first is to include SQL views in your
-database - Datasette will then list those views on your database index page.
+If you want to bundle some pre-written SQL queries with your Datasette-hosted database you can do so in two ways. The first is to include SQL views in your database - Datasette will then list those views on your database index page.
 
 The quickest way to create views is with the SQLite command-line interface::
 
@@ -74,8 +56,7 @@ The quickest way to create views is with the SQLite command-line interface::
 Canned queries
 --------------
 
-As an alternative to adding views to your database, you can define canned
-queries inside your ``metadata.json`` file. Here's an example:
+As an alternative to adding views to your database, you can define canned queries inside your ``metadata.json`` file. Here's an example:
 
 .. code-block:: json
 
@@ -95,8 +76,7 @@ Then run Datasette like this::
 
     datasette sf-trees.db -m metadata.json
 
-Each canned query will be listed on the database index page, and will also get
-its own URL at::
+Each canned query will be listed on the database index page, and will also get its own URL at::
 
     /database-name/canned-query-name
 
@@ -104,20 +84,14 @@ For the above example, that URL would be::
 
     /sf-trees/just_species
 
-You can optionally include ``"title"`` and ``"description"`` keys to show a
-title and description on the canned query page. As with regular table metadata
-you can alternatively specify ``"description_html"`` to have your description
-rendered as HTML (rather than having HTML special characters escaped).
+You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped).
 
 .. _canned_queries_named_parameters:
 
 Canned query parameters
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Canned queries support named parameters, so if you include those in the SQL you
-will then be able to enter them using the form fields on the canned query page
-or by adding them to the URL. This means canned queries can be used to create
-custom JSON APIs based on a carefully designed SQL statement.
+Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement.
 
 Here's an example of a canned query with a named parameter:
 
@@ -380,14 +354,9 @@ The ``"message"`` and ``"redirect"`` values here will take into account ``on_suc
 Pagination
 ----------
 
-Datasette's default table pagination is designed to be extremely efficient. SQL
-OFFSET/LIMIT pagination can have a significant performance penalty once you get
-into multiple thousands of rows, as each page still requires the database to
-scan through every preceding row to find the correct offset.
+Datasette's default table pagination is designed to be extremely efficient. SQL OFFSET/LIMIT pagination can have a significant performance penalty once you get into multiple thousands of rows, as each page still requires the database to scan through every preceding row to find the correct offset.
 
-When paginating through tables, Datasette instead orders the rows in the table
-by their primary key and performs a WHERE clause against the last seen primary
-key for the previous page. For example:
+When paginating through tables, Datasette instead orders the rows in the table by their primary key and performs a WHERE clause against the last seen primary key for the previous page. For example:
 
 .. code-block:: sql
 
@@ -395,15 +364,9 @@ key for the previous page. For example:
 
 This represents page three for this particular table, with a page size of 100.
 
-Note that we request 101 items in the limit clause rather than 100. This allows
-us to detect if we are on the last page of the results: if the query returns
-less than 101 rows we know we have reached the end of the pagination set.
-Datasette will only return the first 100 rows - the 101st is used purely to
-detect if there should be another page.
+Note that we request 101 items in the limit clause rather than 100. This allows us to detect if we are on the last page of the results: if the query returns less than 101 rows we know we have reached the end of the pagination set. Datasette will only return the first 100 rows - the 101st is used purely to detect if there should be another page.
 
-Since the where clause acts against the index on the primary key, the query is
-extremely fast even for records that are a long way into the overall pagination
-set.
+Since the where clause acts against the index on the primary key, the query is extremely fast even for records that are a long way into the overall pagination set.
 
 .. _cross_database_queries:
 

From 15a9d4abfff0c45dee2a9f851326e1d61b1c678c Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 22 Oct 2021 12:34:23 -0700
Subject: [PATCH 0479/1866] Docs on named parameters with cast as real/integer,
 closes #1496

---
 docs/sql_queries.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst
index 4958b56a..f9a36490 100644
--- a/docs/sql_queries.rst
+++ b/docs/sql_queries.rst
@@ -34,6 +34,14 @@ You can also provide values for these fields by constructing a URL::
 
 SQLite string escaping rules will be applied to values passed using named parameters - they will be wrapped in quotes and their content will be correctly escaped.
 
+Values from named parameters are treated as SQLite strings. If you need to perform numeric comparisons on them you should cast them to an integer or float first using ``cast(:name as integer)`` or ``cast(:name as real)``, for example:
+
+.. code-block:: sql
+
+    select * from Street_Tree_List
+    where latitude > cast(:min_latitude as real)
+    and latitude < cast(:max_latitude as real)
+
 Datasette disallows custom SQL queries containing the string PRAGMA (with a small number `of exceptions `__) as SQLite pragma statements can be used to change database settings at runtime. If you need to include the string "pragma" in a query you can do so safely using a named parameter.
 
 .. _sql_views:

From 96a823f2834a262ae97a90ebfb6847f14763c415 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:19:54 -0700
Subject: [PATCH 0480/1866] Fix compatibility with Python 3.10 (#1481)

* Run tests against Python 3.10
* Upgrade to Janus 0.6.2 for Python 3.10
* Add 3.10 to classifiers

Closes #1482
---
 .github/workflows/test.yml | 2 +-
 setup.py                   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bcb241d3..0b3635fe 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
diff --git a/setup.py b/setup.py
index c1b87bd4..2f5fed49 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,7 @@ setup(
         "pluggy>=0.13,<1.1",
         "uvicorn~=0.11",
         "aiofiles>=0.4,<0.8",
-        "janus>=0.4,<0.7",
+        "janus>=0.6.2,<0.7",
         "asgi-csrf>=0.9",
         "PyYAML>=5.3,<7.0",
         "mergedeep>=1.1.1,<1.4.0",
@@ -85,6 +85,7 @@ setup(
         "Intended Audience :: End Users/Desktop",
         "Topic :: Database",
         "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.7",

From 03cc697b6b3d0983618c29ee75b45b5e0ac91139 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 24 Oct 2021 15:22:39 -0700
Subject: [PATCH 0481/1866] Update pytest-asyncio requirement from <0.16,>=0.10
 to >=0.10,<0.17 (#1494)

Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.10.0...v0.16.0)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] 

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2f5fed49..17a56a97 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,7 @@ setup(
         "test": [
             "pytest>=5.2.2,<6.3.0",
             "pytest-xdist>=2.2.1,<2.5",
-            "pytest-asyncio>=0.10,<0.16",
+            "pytest-asyncio>=0.10,<0.17",
             "beautifulsoup4>=4.8.1,<4.11.0",
             "black==21.9b0",
             "pytest-timeout>=1.4.2,<2.1",

From e6e44372b34414eac2f36a4c1120af4f755aa423 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:29:56 -0700
Subject: [PATCH 0482/1866] Release 0.59.1

Refs #1482, #1496
---
 datasette/version.py | 2 +-
 docs/changelog.rst   | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index d78b2e90..ff1f55e8 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59"
+__version__ = "0.59.1"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 24e19baa..40f49fb2 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,14 @@
 Changelog
 =========
 
+.. _v0_59_1:
+
+0.59.1 (2021-10-24)
+-------------------
+
+- Fix compatibility with Python 3.10. (:issue:`1482`)
+- Documentation on how to use :ref:`sql_parameters` with integer and floating point values. (:issue:`1496`)
+
 .. _v0_59:
 
 0.59 (2021-10-14)

From 3a2ed6300d2d31972a5ac633f4e1e9561e163e29 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 15:37:43 -0700
Subject: [PATCH 0483/1866] Run tests on 3.10 during publish, refs #1482

---
 .github/workflows/publish.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 54e582f0..17c6ae9b 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

From 2c31d1cd9cd3b63458ccbe391866499fa3f44978 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sun, 24 Oct 2021 16:24:41 -0700
Subject: [PATCH 0484/1866] Upgrade Docker base to Debian buster, refs #1497

---
 Dockerfile | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7c56cf56..42f5529b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,18 +1,11 @@
-FROM python:3.9.2-slim-buster as build
+FROM python:3.9.7-slim-bullseye as build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55
 ARG VERSION
 
-# software-properties-common provides add-apt-repository
-# which we need in order to install a more recent release
-# of libsqlite3-mod-spatialite from the sid distribution
 RUN apt-get update && \
-    apt-get -y --no-install-recommends install software-properties-common && \
-    add-apt-repository "deb http://httpredir.debian.org/debian sid main" && \
-    apt-get update && \
-    apt-get -t sid install -y --no-install-recommends libsqlite3-mod-spatialite && \
-    apt-get remove -y software-properties-common && \
+    apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \
     apt clean && \
     rm -rf /var/lib/apt && \
     rm -rf /var/lib/dpkg/info/*

From c92ab51b3ce0c2df002c0c2f10549a43910dd4be Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 12 Nov 2021 06:18:31 -0800
Subject: [PATCH 0485/1866] Logo at top of README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ee9d9a5a..ce15ccf4 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Datasette
+Datasette
 
 [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)
 [![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html)

From c306b696de0a582e322f9eb7cb4125c83301e3a9 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 20:44:54 -0800
Subject: [PATCH 0486/1866] Correct facet links for columns with a leading
 underscore, closes #1506

---
 datasette/facets.py              |   9 ++-
 tests/fixtures.py                |  10 +--
 tests/test_api.py                |  24 +++---
 tests/test_csv.py                |   2 +-
 tests/test_facets.py             | 133 ++++++++++++++++++++++++++++++-
 tests/test_html.py               |  32 ++++----
 tests/test_internals_database.py |   4 +-
 tests/test_plugins.py            |   8 +-
 8 files changed, 179 insertions(+), 43 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index f74e2d01..94a1d83d 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -237,14 +237,17 @@ class ColumnFacet(Facet):
                 else:
                     expanded = {}
                 for row in facet_rows:
-                    selected = (column, str(row["value"])) in qs_pairs
+                    column_qs = column
+                    if column.startswith("_"):
+                        column_qs = "{}__exact".format(column)
+                    selected = (column_qs, str(row["value"])) in qs_pairs
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column: str(row["value"])}
+                            self.request, {column_qs: str(row["value"])}
                         )
                     else:
                         toggle_path = path_with_added_args(
-                            self.request, {column: row["value"]}
+                            self.request, {column_qs: row["value"]}
                         )
                     facet_results_values.append(
                         {
diff --git a/tests/fixtures.py b/tests/fixtures.py
index dc22c609..1a879126 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -355,12 +355,12 @@ METADATA = {
                 "neighborhood_search": {
                     "sql": textwrap.dedent(
                         """
-                        select neighborhood, facet_cities.name, state
+                        select _neighborhood, facet_cities.name, state
                         from facetable
                             join facet_cities
                                 on facetable.city_id = facet_cities.id
-                        where neighborhood like '%' || :text || '%'
-                        order by neighborhood;
+                        where _neighborhood like '%' || :text || '%'
+                        order by _neighborhood;
                     """
                     ),
                     "title": "Search neighborhoods",
@@ -559,14 +559,14 @@ CREATE TABLE facetable (
     on_earth integer,
     state text,
     city_id integer,
-    neighborhood text,
+    _neighborhood text,
     tags text,
     complex_array text,
     distinct_some_null,
     FOREIGN KEY ("city_id") REFERENCES [facet_cities](id)
 );
 INSERT INTO facetable
-    (created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null)
+    (created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null)
 VALUES
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one'),
     ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two'),
diff --git a/tests/test_api.py b/tests/test_api.py
index 311ae464..43b52175 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -213,7 +213,7 @@ def test_database_page(app_client):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -1241,7 +1241,9 @@ def test_table_filter_json_arraynotcontains(app_client):
 
 
 def test_table_filter_extra_where(app_client):
-    response = app_client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'")
+    response = app_client.get(
+        "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+    )
     assert [
         [
             2,
@@ -1259,14 +1261,16 @@ def test_table_filter_extra_where(app_client):
 
 
 def test_table_filter_extra_where_invalid(app_client):
-    response = app_client.get("/fixtures/facetable.json?_where=neighborhood=Dogpatch'")
+    response = app_client.get("/fixtures/facetable.json?_where=_neighborhood=Dogpatch'")
     assert 400 == response.status
     assert "Invalid SQL" == response.json["title"]
 
 
 def test_table_filter_extra_where_disabled_if_no_sql_allowed():
     with make_app_client(metadata={"allow_sql": {}}) as client:
-        response = client.get("/fixtures/facetable.json?_where=neighborhood='Dogpatch'")
+        response = client.get(
+            "/fixtures/facetable.json?_where=_neighborhood='Dogpatch'"
+        )
         assert 403 == response.status
         assert "_where= is not allowed" == response.json["error"]
 
@@ -1696,7 +1700,7 @@ def test_suggested_facets(app_client):
         {"name": "on_earth", "querystring": "_facet=on_earth"},
         {"name": "state", "querystring": "_facet=state"},
         {"name": "city_id", "querystring": "_facet=city_id"},
-        {"name": "neighborhood", "querystring": "_facet=neighborhood"},
+        {"name": "_neighborhood", "querystring": "_facet=_neighborhood"},
         {"name": "tags", "querystring": "_facet=tags"},
         {"name": "complex_array", "querystring": "_facet=complex_array"},
         {"name": "created", "querystring": "_facet_date=created"},
@@ -1752,7 +1756,7 @@ def test_nocount_nofacet_if_shape_is_object(app_client_with_trace):
 def test_expand_labels(app_client):
     response = app_client.get(
         "/fixtures/facetable.json?_shape=object&_labels=1&_size=2"
-        "&neighborhood__contains=c"
+        "&_neighborhood__contains=c"
     )
     assert {
         "2": {
@@ -1762,7 +1766,7 @@ def test_expand_labels(app_client):
             "on_earth": 1,
             "state": "CA",
             "city_id": {"value": 1, "label": "San Francisco"},
-            "neighborhood": "Dogpatch",
+            "_neighborhood": "Dogpatch",
             "tags": '["tag1", "tag3"]',
             "complex_array": "[]",
             "distinct_some_null": "two",
@@ -1774,7 +1778,7 @@ def test_expand_labels(app_client):
             "on_earth": 1,
             "state": "MI",
             "city_id": {"value": 3, "label": "Detroit"},
-            "neighborhood": "Corktown",
+            "_neighborhood": "Corktown",
             "tags": "[]",
             "complex_array": "[]",
             "distinct_some_null": None,
@@ -2125,7 +2129,7 @@ def test_http_options_request(app_client):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -2152,7 +2156,7 @@ def test_http_options_request(app_client):
                 "planet_int",
                 "on_earth",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
diff --git a/tests/test_csv.py b/tests/test_csv.py
index 5e9406e7..5902e9db 100644
--- a/tests/test_csv.py
+++ b/tests/test_csv.py
@@ -24,7 +24,7 @@ world
 )
 
 EXPECTED_TABLE_WITH_LABELS_CSV = """
-pk,created,planet_int,on_earth,state,city_id,city_id_label,neighborhood,tags,complex_array,distinct_some_null
+pk,created,planet_int,on_earth,state,city_id,city_id_label,_neighborhood,tags,complex_array,distinct_some_null
 1,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Mission,"[""tag1"", ""tag2""]","[{""foo"": ""bar""}]",one
 2,2019-01-14 08:00:00,1,1,CA,1,San Francisco,Dogpatch,"[""tag1"", ""tag3""]",[],two
 3,2019-01-14 08:00:00,1,1,CA,1,San Francisco,SOMA,[],[],
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 22927512..01d8b8f5 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -23,7 +23,10 @@ async def test_column_facet_suggest(app_client):
         {"name": "on_earth", "toggle_url": "http://localhost/?_facet=on_earth"},
         {"name": "state", "toggle_url": "http://localhost/?_facet=state"},
         {"name": "city_id", "toggle_url": "http://localhost/?_facet=city_id"},
-        {"name": "neighborhood", "toggle_url": "http://localhost/?_facet=neighborhood"},
+        {
+            "name": "_neighborhood",
+            "toggle_url": "http://localhost/?_facet=_neighborhood",
+        },
         {"name": "tags", "toggle_url": "http://localhost/?_facet=tags"},
         {
             "name": "complex_array",
@@ -56,8 +59,8 @@ async def test_column_facet_suggest_skip_if_already_selected(app_client):
             "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=city_id",
         },
         {
-            "name": "neighborhood",
-            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=neighborhood",
+            "name": "_neighborhood",
+            "toggle_url": "http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_neighborhood",
         },
         {
             "name": "tags",
@@ -86,7 +89,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(app_client):
         "planet_int",
         "on_earth",
         "state",
-        "neighborhood",
+        "_neighborhood",
         "tags",
         "complex_array",
     ] == suggestions
@@ -144,6 +147,128 @@ async def test_column_facet_results(app_client):
     } == buckets
 
 
+@pytest.mark.asyncio
+async def test_column_facet_results_column_starts_with_underscore(app_client):
+    facet = ColumnFacet(
+        app_client.ds,
+        Request.fake("/?_facet=_neighborhood"),
+        database="fixtures",
+        sql="select * from facetable",
+        table="facetable",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert [] == timed_out
+    assert buckets == {
+        "_neighborhood": {
+            "name": "_neighborhood",
+            "type": "column",
+            "hideable": True,
+            "toggle_url": "/",
+            "results": [
+                {
+                    "value": "Downtown",
+                    "label": "Downtown",
+                    "count": 2,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Downtown",
+                    "selected": False,
+                },
+                {
+                    "value": "Arcadia Planitia",
+                    "label": "Arcadia Planitia",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Arcadia+Planitia",
+                    "selected": False,
+                },
+                {
+                    "value": "Bernal Heights",
+                    "label": "Bernal Heights",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Bernal+Heights",
+                    "selected": False,
+                },
+                {
+                    "value": "Corktown",
+                    "label": "Corktown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Corktown",
+                    "selected": False,
+                },
+                {
+                    "value": "Dogpatch",
+                    "label": "Dogpatch",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Dogpatch",
+                    "selected": False,
+                },
+                {
+                    "value": "Greektown",
+                    "label": "Greektown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Greektown",
+                    "selected": False,
+                },
+                {
+                    "value": "Hayes Valley",
+                    "label": "Hayes Valley",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hayes+Valley",
+                    "selected": False,
+                },
+                {
+                    "value": "Hollywood",
+                    "label": "Hollywood",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hollywood",
+                    "selected": False,
+                },
+                {
+                    "value": "Koreatown",
+                    "label": "Koreatown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Koreatown",
+                    "selected": False,
+                },
+                {
+                    "value": "Los Feliz",
+                    "label": "Los Feliz",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Los+Feliz",
+                    "selected": False,
+                },
+                {
+                    "value": "Mexicantown",
+                    "label": "Mexicantown",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mexicantown",
+                    "selected": False,
+                },
+                {
+                    "value": "Mission",
+                    "label": "Mission",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mission",
+                    "selected": False,
+                },
+                {
+                    "value": "SOMA",
+                    "label": "SOMA",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=SOMA",
+                    "selected": False,
+                },
+                {
+                    "value": "Tenderloin",
+                    "label": "Tenderloin",
+                    "count": 1,
+                    "toggle_url": "http://localhost/?_facet=_neighborhood&_neighborhood__exact=Tenderloin",
+                    "selected": False,
+                },
+            ],
+            "truncated": False,
+        }
+    }
+
+
 @pytest.mark.asyncio
 async def test_column_facet_from_metadata_cannot_be_hidden(app_client):
     facet = ColumnFacet(
diff --git a/tests/test_html.py b/tests/test_html.py
index 5f2ba2f1..1955e65b 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -235,7 +235,10 @@ def test_table_cell_truncation():
             "Corkt…",
             "Mexic…",
             "Arcad…",
-        ] == [td.string for td in table.findAll("td", {"class": "col-neighborhood"})]
+        ] == [
+            td.string
+            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
+        ]
 
 
 def test_row_page_does_not_truncate():
@@ -245,7 +248,8 @@ def test_row_page_does_not_truncate():
         table = Soup(response.body, "html.parser").find("table")
         assert table["class"] == ["rows-and-columns"]
         assert ["Mission"] == [
-            td.string for td in table.findAll("td", {"class": "col-neighborhood"})
+            td.string
+            for td in table.findAll("td", {"class": "col-neighborhood-b352a7"})
         ]
 
 
@@ -1312,7 +1316,7 @@ def test_canned_query_show_hide_metadata_option(
 
 def test_extra_where_clauses(app_client):
     response = app_client.get(
-        "/fixtures/facetable?_where=neighborhood='Dogpatch'&_where=city_id=1"
+        "/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=city_id=1"
     )
     soup = Soup(response.body, "html.parser")
     div = soup.select(".extra-wheres")[0]
@@ -1320,12 +1324,12 @@ def test_extra_where_clauses(app_client):
     hrefs = [a["href"] for a in div.findAll("a")]
     assert [
         "/fixtures/facetable?_where=city_id%3D1",
-        "/fixtures/facetable?_where=neighborhood%3D%27Dogpatch%27",
+        "/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27",
     ] == hrefs
     # These should also be persisted as hidden fields
     inputs = soup.find("form").findAll("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]
-    assert [("_where", "neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [
+    assert [("_where", "_neighborhood='Dogpatch'"), ("_where", "city_id=1")] == [
         (hidden["name"], hidden["value"]) for hidden in hiddens
     ]
 
@@ -1634,11 +1638,11 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):
     [
         (
             "/fixtures/neighborhood_search",
-            "/fixtures?sql=%0Aselect+neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+neighborhood%3B%0A&text=",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=",
         ),
         (
             "/fixtures/neighborhood_search?text=ber",
-            "/fixtures?sql=%0Aselect+neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+neighborhood%3B%0A&text=ber",
+            "/fixtures?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable.city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber",
         ),
         ("/fixtures/pragma_cache_size", None),
         (
@@ -1716,23 +1720,23 @@ def test_navigation_menu_links(
         (
             5,
             # Default should show 2 facets
-            "/fixtures/facetable?_facet=neighborhood",
+            "/fixtures/facetable?_facet=_neighborhood",
             2,
             True,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
         ),
         # _facet_size above max_returned_rows should show max_returned_rows (5)
         (
             5,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=50",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=50",
             5,
             True,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
         ),
         # If max_returned_rows is high enough, should return all
         (
             20,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
             14,
             False,
             None,
@@ -1741,7 +1745,7 @@ def test_navigation_menu_links(
         # _facet_size above max_returned_rows should show max_returned_rows (5)
         (
             5,
-            "/fixtures/facetable?_facet=neighborhood&_facet_size=max",
+            "/fixtures/facetable?_facet=_neighborhood&_facet_size=max",
             5,
             True,
             None,
@@ -1760,7 +1764,7 @@ def test_facet_more_links(
     ) as client:
         response = client.get(path)
         soup = Soup(response.body, "html.parser")
-        lis = soup.select("#facet-neighborhood ul li:not(.facet-truncated)")
+        lis = soup.select("#facet-neighborhood-b352a7 ul li:not(.facet-truncated)")
         facet_truncated = soup.select_one(".facet-truncated")
         assert len(lis) == expected_num_facets
         if not expected_ellipses:
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index ad829751..2d0cae7f 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -82,7 +82,7 @@ async def test_table_exists(db, tables, exists):
                 "on_earth",
                 "state",
                 "city_id",
-                "neighborhood",
+                "_neighborhood",
                 "tags",
                 "complex_array",
                 "distinct_some_null",
@@ -170,7 +170,7 @@ async def test_table_columns(db, table, expected):
                 ),
                 Column(
                     cid=6,
-                    name="neighborhood",
+                    name="_neighborhood",
                     type="text",
                     notnull=0,
                     default_value=None,
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 7dac8002..7ac6b173 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -437,7 +437,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "on_earth",
             "state",
             "city_id",
-            "neighborhood",
+            "_neighborhood",
             "tags",
             "complex_array",
             "distinct_some_null",
@@ -459,7 +459,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
             "",
             "",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
@@ -526,12 +526,12 @@ def test_hook_register_output_renderer_can_render(app_client):
             "on_earth",
             "state",
             "city_id",
-            "neighborhood",
+            "_neighborhood",
             "tags",
             "complex_array",
             "distinct_some_null",
         ],
-        "sql": "select pk, created, planet_int, on_earth, state, city_id, neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
+        "sql": "select pk, created, planet_int, on_earth, state, city_id, _neighborhood, tags, complex_array, distinct_some_null from facetable order by pk limit 51",
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",

From c9e3cfecc8e966e5137d72e3f2150be9602d55f5 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 20:53:00 -0800
Subject: [PATCH 0487/1866] Columns in filters now ignore ?_nocol, closes #1503

---
 datasette/views/table.py |  2 +-
 tests/test_html.py       | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index efcef4d2..3e8f38f6 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -922,7 +922,7 @@ class TableView(RowTableShared):
                 "use_rowid": use_rowid,
                 "filters": filters,
                 "display_columns": display_columns,
-                "filter_columns": columns,
+                "filter_columns": table_columns,
                 "display_rows": display_rows,
                 "facets_timed_out": facets_timed_out,
                 "sorted_facet_results": sorted(
diff --git a/tests/test_html.py b/tests/test_html.py
index 1955e65b..eb1e3d20 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -923,6 +923,26 @@ def test_table_html_filter_form_column_options(
     assert expected_column_options == column_options
 
 
+def test_table_html_filter_form_still_shows_nocol_columns(app_client):
+    # https://github.com/simonw/datasette/issues/1503
+    response = app_client.get("/fixtures/sortable?_nocol=sortable")
+    assert response.status == 200
+    form = Soup(response.body, "html.parser").find("form")
+    assert [
+        o.string
+        for o in form.select("select[name='_filter_column']")[0].select("option")
+    ] == [
+        "- column -",
+        "pk1",
+        "pk2",
+        "content",
+        "sortable",
+        "sortable_with_nulls",
+        "sortable_with_nulls_2",
+        "text",
+    ]
+
+
 def test_row_html_compound_primary_key(app_client):
     response = app_client.get("/fixtures/compound_primary_key/a,b")
     assert response.status == 200

From 1c13e1af0664a4dfb1e69714c56523279cae09e4 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:08:33 -0800
Subject: [PATCH 0488/1866] Ensure query columns are included too, ref #1503

---
 datasette/views/table.py | 10 +++++++++-
 tests/test_html.py       |  3 ++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/datasette/views/table.py b/datasette/views/table.py
index 3e8f38f6..e6ae67de 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -915,6 +915,14 @@ class TableView(RowTableShared):
                         links.extend(extra_links)
                 return links
 
+            # filter_columns combine the columns we know are available
+            # in the table with any additional columns (such as rowid)
+            # which are available in the query
+            filter_columns = list(columns) + [
+                table_column
+                for table_column in table_columns
+                if table_column not in columns
+            ]
             return {
                 "table_actions": table_actions,
                 "supports_search": bool(fts_table),
@@ -922,7 +930,7 @@ class TableView(RowTableShared):
                 "use_rowid": use_rowid,
                 "filters": filters,
                 "display_columns": display_columns,
-                "filter_columns": table_columns,
+                "filter_columns": filter_columns,
                 "display_rows": display_rows,
                 "facets_timed_out": facets_timed_out,
                 "sorted_facet_results": sorted(
diff --git a/tests/test_html.py b/tests/test_html.py
index eb1e3d20..f24165bd 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -936,10 +936,11 @@ def test_table_html_filter_form_still_shows_nocol_columns(app_client):
         "pk1",
         "pk2",
         "content",
-        "sortable",
         "sortable_with_nulls",
         "sortable_with_nulls_2",
         "text",
+        # Moved to the end because it is no longer returned by the query:
+        "sortable",
     ]
 
 

From de1e031713f47fbd51eb7239db3e7e6025fbf81a Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:14:43 -0800
Subject: [PATCH 0489/1866] Release 0.59.2

Refs #1497, #1503, #1506
---
 datasette/version.py |  2 +-
 docs/changelog.rst   | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/datasette/version.py b/datasette/version.py
index ff1f55e8..db89b418 100644
--- a/datasette/version.py
+++ b/datasette/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.59.1"
+__version__ = "0.59.2"
 __version_info__ = tuple(__version__.split("."))
diff --git a/docs/changelog.rst b/docs/changelog.rst
index 40f49fb2..47ca3480 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -4,6 +4,16 @@
 Changelog
 =========
 
+.. _v0_59_2:
+
+0.59.2 (2021-11-13)
+-------------------
+
+- Column names with a leading underscore now work correctly when used as a facet. (:issue:`1506`)
+- Applying ``?_nocol=`` to a column no longer removes that column from the filtering interface. (:issue:`1503`)
+- Official Datasette Docker container now uses Debian Bullseye as the base image. (:issue:`1497`)
+- Datasette is four years old today! Here's the `original release announcement `__ from 2017.
+
 .. _v0_59_1:
 
 0.59.1 (2021-10-24)

From 030390fd4abcecf1ab80d0528e32d7dbc50d1b5f Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:29:43 -0800
Subject: [PATCH 0490/1866] .readthedocs.yaml configuration, refs #1507

---
 .readthedocs.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 .readthedocs.yaml

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..816c10e2
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,9 @@
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
+sphinx:
+   configuration: docs/conf.py

From 502c02fa6dde6a8bb840af6c4c8cf858aa1db687 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Sat, 13 Nov 2021 21:37:40 -0800
Subject: [PATCH 0491/1866] Pin to docutils<0.18 in ReadTheDocs, refs #1507

---
 .readthedocs.yaml                 | 4 ++++
 docs/readthedocs-requirements.txt | 1 +
 2 files changed, 5 insertions(+)
 create mode 100644 docs/readthedocs-requirements.txt

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 816c10e2..70db5313 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -7,3 +7,7 @@ build:
 
 sphinx:
    configuration: docs/conf.py
+
+python:
+   install:
+   - requirements: docs/readthedocs-requirements.txt
diff --git a/docs/readthedocs-requirements.txt b/docs/readthedocs-requirements.txt
new file mode 100644
index 00000000..93120e66
--- /dev/null
+++ b/docs/readthedocs-requirements.txt
@@ -0,0 +1 @@
+docutils<0.18

From 07044bd130542870d5eb2e545988d0a24eb573ec Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 15:41:07 -0800
Subject: [PATCH 0492/1866] SQL view-friendly arraycontains/arraynotcontains
 implementation, refs #448

---
 datasette/filters.py  | 10 ++--------
 tests/test_filters.py | 11 +++++++----
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/datasette/filters.py b/datasette/filters.py
index 2b859d99..cbd94415 100644
--- a/datasette/filters.py
+++ b/datasette/filters.py
@@ -149,19 +149,13 @@ class Filters:
                 TemplatedFilter(
                     "arraycontains",
                     "array contains",
-                    """rowid in (
-            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
-            where j.value = :{p}
-        )""",
+                    """:{p} in (select value from json_each([{t}].[{c}]))""",
                     '{c} contains "{v}"',
                 ),
                 TemplatedFilter(
                     "arraynotcontains",
                     "array does not contain",
-                    """rowid not in (
-            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
-            where j.value = :{p}
-        )""",
+                    """:{p} not in (select value from json_each([{t}].[{c}]))""",
                     '{c} does not contain "{v}"',
                 ),
             ]
diff --git a/tests/test_filters.py b/tests/test_filters.py
index f22b7b5c..d05ae80f 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -56,12 +56,15 @@ import pytest
         # Not in, and JSON array not in
         ((("foo__notin", "1,2,3"),), ["foo not in (:p0, :p1, :p2)"], ["1", "2", "3"]),
         ((("foo__notin", "[1,2,3]"),), ["foo not in (:p0, :p1, :p2)"], [1, 2, 3]),
-        # JSON arraycontains
+        # JSON arraycontains, arraynotcontains
         (
             (("Availability+Info__arraycontains", "yes"),),
-            [
-                "rowid in (\n            select table.rowid from table, json_each([table].[Availability+Info]) j\n            where j.value = :p0\n        )"
-            ],
+            [":p0 in (select value from json_each([table].[Availability+Info]))"],
+            ["yes"],
+        ),
+        (
+            (("Availability+Info__arraynotcontains", "yes"),),
+            [":p0 not in (select value from json_each([table].[Availability+Info]))"],
             ["yes"],
         ),
     ],

From 55024b5301892306b786fc37a8ab3c096be5c227 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 17:19:33 -0800
Subject: [PATCH 0493/1866] _facet_array no longer confused by duplicate array
 items, closes #448

---
 datasette/facets.py  | 23 +++++++++++++++---
 tests/test_facets.py | 58 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 94a1d83d..ce2111c9 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -354,11 +354,26 @@ class ArrayFacet(Facet):
             config = source_and_config["config"]
             source = source_and_config["source"]
             column = config.get("column") or config["simple"]
+            # https://github.com/simonw/datasette/issues/448
             facet_sql = """
-                select j.value as value, count(*) as count from (
-                    {sql}
-                ) join json_each({col}) j
-                group by j.value order by count desc, value limit {limit}
+                with inner as ({sql}),
+                deduped_array_items as (
+                    select
+                        distinct j.value,
+                        inner.*
+                    from
+                        json_each([inner].{col}) j
+                        join inner
+                )
+                select
+                    value as value,
+                    count(*) as count
+                from
+                    deduped_array_items
+                group by
+                    value
+                order by
+                    count(*) desc limit {limit}
             """.format(
                 col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
             )
diff --git a/tests/test_facets.py b/tests/test_facets.py
index 01d8b8f5..a20c79c4 100644
--- a/tests/test_facets.py
+++ b/tests/test_facets.py
@@ -4,6 +4,7 @@ from datasette.facets import ColumnFacet, ArrayFacet, DateFacet
 from datasette.utils.asgi import Request
 from datasette.utils import detect_json1
 from .fixtures import app_client  # noqa
+import json
 import pytest
 
 
@@ -402,6 +403,63 @@ async def test_array_facet_results(app_client):
     } == buckets
 
 
+@pytest.mark.asyncio
+@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
+async def test_array_facet_handle_duplicate_tags():
+    ds = Datasette([], memory=True)
+    db = ds.add_database(Database(ds, memory_name="test_array_facet"))
+    await db.execute_write("create table otters(name text, tags text)", block=True)
+    for name, tags in (
+        ("Charles", ["friendly", "cunning", "friendly"]),
+        ("Shaun", ["cunning", "empathetic", "friendly"]),
+        ("Tracy", ["empathetic", "eager"]),
+    ):
+        await db.execute_write(
+            "insert into otters (name, tags) values (?, ?)",
+            [name, json.dumps(tags)],
+            block=True,
+        )
+
+    response = await ds.client.get("/test_array_facet/otters.json?_facet_array=tags")
+    assert response.json()["facet_results"]["tags"] == {
+        "name": "tags",
+        "type": "array",
+        "results": [
+            {
+                "value": "cunning",
+                "label": "cunning",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=cunning",
+                "selected": False,
+            },
+            {
+                "value": "empathetic",
+                "label": "empathetic",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=empathetic",
+                "selected": False,
+            },
+            {
+                "value": "friendly",
+                "label": "friendly",
+                "count": 2,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=friendly",
+                "selected": False,
+            },
+            {
+                "value": "eager",
+                "label": "eager",
+                "count": 1,
+                "toggle_url": "http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=eager",
+                "selected": False,
+            },
+        ],
+        "hideable": True,
+        "toggle_url": "/test_array_facet/otters.json",
+        "truncated": False,
+    }
+
+
 @pytest.mark.asyncio
 async def test_date_facet_results(app_client):
     facet = DateFacet(

From 0156c6b5e52d541e93f0d68e9245f20ae83bc933 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Mon, 15 Nov 2021 17:31:33 -0800
Subject: [PATCH 0494/1866] Facet in predictable order for tests, refs #448

---
 datasette/facets.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index ce2111c9..62e7775e 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -373,7 +373,7 @@ class ArrayFacet(Facet):
                 group by
                     value
                 order by
-                    count(*) desc limit {limit}
+                    count(*) desc, value limit {limit}
             """.format(
                 col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
             )

From 6e971b4ac175df95ac7fe5dc2b57b53ad7f533fc Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Nov 2021 19:07:21 -0800
Subject: [PATCH 0495/1866] Test confirming plugins can over-ride default
 routes, closes #1517

---
 tests/plugins/my_plugin_2.py | 10 +++++++++-
 tests/test_plugins.py        | 19 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py
index ba298fd4..f5ce36b3 100644
--- a/tests/plugins/my_plugin_2.py
+++ b/tests/plugins/my_plugin_2.py
@@ -176,4 +176,12 @@ def register_routes(datasette):
     if not config:
         return
     path = config["path"]
-    return [(r"/{}/$".format(path), lambda: Response.text(path.upper()))]
+
+    def new_table(request):
+        return Response.text("/db/table: {}".format(sorted(request.url_vars.items())))
+
+    return [
+        (r"/{}/$".format(path), lambda: Response.text(path.upper())),
+        # Also serves to demonstrate over-ride of default paths:
+        (r"/(?P[^/]+)/(?P[^/]+?$)", new_table),
+    ]
diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 7ac6b173..c9ff6edb 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -678,6 +678,25 @@ def test_hook_register_routes_with_datasette(configured_path):
         assert client.get(f"/{other_path}/", follow_redirects=True).status == 404
 
 
+def test_hook_register_routes_override():
+    "Plugins can over-ride default paths such as /db/table"
+    with make_app_client(
+        metadata={
+            "plugins": {
+                "register-route-demo": {
+                    "path": "blah",
+                }
+            }
+        }
+    ) as client:
+        response = client.get("/db/table")
+        assert response.status == 200
+        assert (
+            response.text
+            == "/db/table: [('db_name', 'db'), ('table_and_format', 'table')]"
+        )
+
+
 def test_hook_register_routes_post(app_client):
     response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True)
     assert 200 == response.status

From 30255055150d7bc0affc8156adc18295495020ff Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Thu, 18 Nov 2021 19:19:43 -0800
Subject: [PATCH 0496/1866] functools.wraps to help investigate #1517

---
 datasette/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/datasette/app.py b/datasette/app.py
index 52c5e629..28268e42 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -2,6 +2,7 @@ import asyncio
 import asgi_csrf
 import collections
 import datetime
+import functools
 import glob
 import hashlib
 import httpx
@@ -1354,6 +1355,7 @@ def _cleaner_task_str(task):
 
 
 def wrap_view(view_fn, datasette):
+    @functools.wraps(view_fn)
     async def async_view_fn(request, send):
         if inspect.iscoroutinefunction(view_fn):
             response = await async_call_with_supported_arguments(

From ff0dd4da38d48c2fa9250ecf336002c9ed724e36 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 12:29:37 -0800
Subject: [PATCH 0497/1866] repr() method for Request, refs #1519

---
 datasette/utils/asgi.py         | 3 +++
 tests/test_internals_request.py | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py
index 696944df..ad137fa9 100644
--- a/datasette/utils/asgi.py
+++ b/datasette/utils/asgi.py
@@ -37,6 +37,9 @@ class Request:
         self.scope = scope
         self.receive = receive
 
+    def __repr__(self):
+        return ''.format(self.method, self.url)
+
     @property
     def method(self):
         return self.scope["method"]
diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py
index c42cfbd3..cd956f3f 100644
--- a/tests/test_internals_request.py
+++ b/tests/test_internals_request.py
@@ -75,6 +75,14 @@ def test_request_args():
         request.args["missing"]
 
 
+def test_request_repr():
+    request = Request.fake("/foo?multi=1&multi=2&single=3")
+    assert (
+        repr(request)
+        == ''
+    )
+
+
 def test_request_url_vars():
     scope = {
         "http_version": "1.1",

From c76bbd40664f789c45564b7796628e5110cd3b17 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 14:50:06 -0800
Subject: [PATCH 0498/1866] New live demo with Apache proxying, refs #1522

---
 demos/apache-proxy/Dockerfile | 42 +++++++++++++++++++++++++++++++++++
 demos/apache-proxy/README.md  |  5 +++++
 demos/apache-proxy/deploy.sh  | 13 +++++++++++
 3 files changed, 60 insertions(+)
 create mode 100644 demos/apache-proxy/Dockerfile
 create mode 100644 demos/apache-proxy/README.md
 create mode 100755 demos/apache-proxy/deploy.sh

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
new file mode 100644
index 00000000..2956f913
--- /dev/null
+++ b/demos/apache-proxy/Dockerfile
@@ -0,0 +1,42 @@
+FROM python:3-alpine
+
+RUN apk add --no-cache \
+	apache2 \
+	apache2-proxy \
+	bash
+
+RUN pip install datasette
+
+ENV TINI_VERSION v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
+RUN chmod +x /tini
+
+# Append this to the end of the default httpd.conf file
+RUN echo $'ServerName localhost\n\
+\n\
+\n\
+  Order deny,allow\n\
+  Allow from all\n\
+\n\
+\n\
+ProxyPass        /prefix/ http://localhost:8001/\n\
+Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
+
+RUN echo $'Datasette' > /var/www/localhost/htdocs/index.html
+
+WORKDIR /app
+
+ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
+
+RUN echo $'#!/usr/bin/env bash\n\
+set -e\n\
+\n\
+httpd -D FOREGROUND &\n\
+datasette fixtures.db --setting base_url "/prefix/" -h 0.0.0.0 -p 8001 &\n\
+\n\
+wait -n' > /app/start.sh
+
+RUN chmod +x /app/start.sh
+
+EXPOSE 80
+ENTRYPOINT ["/tini", "--", "/app/start.sh"]
diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md
new file mode 100644
index 00000000..9bd3897c
--- /dev/null
+++ b/demos/apache-proxy/README.md
@@ -0,0 +1,5 @@
+# Datasette running behind an Apache proxy
+
+See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)
+
+This live demo is running at https://apache-proxy-demo.datasette.io/
diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy.sh
new file mode 100755
index 00000000..ae33941c
--- /dev/null
+++ b/demos/apache-proxy/deploy.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# https://til.simonwillison.net/cloudrun/ship-dockerfile-to-cloud-run
+
+NAME="datasette-apache-proxy-demo"
+PROJECT=$(gcloud config get-value project)
+IMAGE="gcr.io/$PROJECT/$NAME"
+
+gcloud builds submit --tag $IMAGE
+gcloud run deploy \
+    --allow-unauthenticated \
+    --platform=managed \
+    --image $IMAGE $NAME \
+    --port 80

From c617e1769ea27e045b0f2907ef49a9a1244e577d Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 15:13:17 -0800
Subject: [PATCH 0499/1866] Fixed test I broke with new repr() in ##1519

---
 tests/test_plugins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index c9ff6edb..697a6b32 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -463,7 +463,7 @@ def test_hook_register_output_renderer_all_parameters(app_client):
         "query_name": None,
         "database": "fixtures",
         "table": "facetable",
-        "request": "",
+        "request": '',
         "view_name": "table",
         "1+1": 2,
     }

From a1ba6cd6bb86d935cdad240de6be6b37aad683f2 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 16:34:35 -0800
Subject: [PATCH 0500/1866] Use build arguments, refs #1522

---
 demos/apache-proxy/Dockerfile | 23 +++++++++++++----------
 demos/apache-proxy/README.md  | 11 +++++++++++
 demos/apache-proxy/deploy.sh  | 31 ++++++++++++++++++++++++-------
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile
index 2956f913..46697c63 100644
--- a/demos/apache-proxy/Dockerfile
+++ b/demos/apache-proxy/Dockerfile
@@ -5,38 +5,41 @@ RUN apk add --no-cache \
 	apache2-proxy \
 	bash
 
-RUN pip install datasette
+ARG DATASETTE_REF
+
+RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip
 
 ENV TINI_VERSION v0.18.0
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini
 RUN chmod +x /tini
 
 # Append this to the end of the default httpd.conf file
-RUN echo $'ServerName localhost\n\
+RUN echo -e 'ServerName localhost\n\
 \n\
 \n\
   Order deny,allow\n\
   Allow from all\n\
 \n\
 \n\
-ProxyPass        /prefix/ http://localhost:8001/\n\
+ProxyPass /prefix/ http://localhost:8001/\n\
 Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf
 
-RUN echo $'Datasette' > /var/www/localhost/htdocs/index.html
+RUN echo 'Datasette' > /var/www/localhost/htdocs/index.html
 
 WORKDIR /app
 
 ADD https://latest.datasette.io/fixtures.db /app/fixtures.db
 
-RUN echo $'#!/usr/bin/env bash\n\
-set -e\n\
+RUN echo -e "#!/usr/bin/env bash\n\
+datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001 &\n\
 \n\
-httpd -D FOREGROUND &\n\
-datasette fixtures.db --setting base_url "/prefix/" -h 0.0.0.0 -p 8001 &\n\
+httpd -D FOREGROUND & \n\
 \n\
-wait -n' > /app/start.sh
+wait -n\n\
+exit $?" > /app/start.sh
 
 RUN chmod +x /app/start.sh
 
 EXPOSE 80
-ENTRYPOINT ["/tini", "--", "/app/start.sh"]
+
+CMD /tini -- /app/start.sh
diff --git a/demos/apache-proxy/README.md b/demos/apache-proxy/README.md
index 9bd3897c..08048512 100644
--- a/demos/apache-proxy/README.md
+++ b/demos/apache-proxy/README.md
@@ -3,3 +3,14 @@
 See also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)
 
 This live demo is running at https://apache-proxy-demo.datasette.io/
+
+To build locally, passing in a Datasette commit hash (or `main` for the main branch):
+
+    docker build -t datasette-apache-proxy-demo . \
+      --build-arg DATASETTE_REF=c617e1769ea27e045b0f2907ef49a9a1244e577d
+
+Then run it like this:
+
+    docker run -p 5000:80 datasette-apache-proxy-demo
+
+And visit `http://localhost:5000/` or `http://localhost:5000/prefix/`
diff --git a/demos/apache-proxy/deploy.sh b/demos/apache-proxy/deploy.sh
index ae33941c..2846590a 100755
--- a/demos/apache-proxy/deploy.sh
+++ b/demos/apache-proxy/deploy.sh
@@ -1,13 +1,30 @@
 #!/bin/bash
-# https://til.simonwillison.net/cloudrun/ship-dockerfile-to-cloud-run
+# https://til.simonwillison.net/cloudrun/using-build-args-with-cloud-run
+
+if [[ -z "$DATASETTE_REF" ]]; then
+    echo "Must provide DATASETTE_REF environment variable" 1>&2
+    exit 1
+fi
 
 NAME="datasette-apache-proxy-demo"
 PROJECT=$(gcloud config get-value project)
 IMAGE="gcr.io/$PROJECT/$NAME"
 
-gcloud builds submit --tag $IMAGE
-gcloud run deploy \
-    --allow-unauthenticated \
-    --platform=managed \
-    --image $IMAGE $NAME \
-    --port 80
+# Need YAML so we can set --build-arg
+echo "
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['build', '-t', '$IMAGE', '.', '--build-arg', 'DATASETTE_REF=$DATASETTE_REF']
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['push', '$IMAGE']
+" > /tmp/cloudbuild.yml
+
+gcloud builds submit --config /tmp/cloudbuild.yml
+
+rm /tmp/cloudbuild.yml
+
+gcloud run deploy $NAME \
+  --allow-unauthenticated \
+  --platform=managed \
+  --image $IMAGE \
+  --port 80

From fe687fd0207c4c56c4778d3e92e3505fc4b18172 Mon Sep 17 00:00:00 2001
From: Simon Willison 
Date: Fri, 19 Nov 2021 16:52:33 -0800
Subject: [PATCH 0501/1866] Fixed a whole bunch of broken base_url links

Refs #1519, #838
---
 datasette/facets.py             | 18 +++++++++++++-----
 datasette/templates/_table.html |  4 ++--
 datasette/views/base.py         | 10 ++++++----
 datasette/views/database.py     |  2 +-
 datasette/views/table.py        |  1 +
 tests/test_html.py              | 24 +++++++++++++++++-------
 6 files changed, 40 insertions(+), 19 deletions(-)

diff --git a/datasette/facets.py b/datasette/facets.py
index 62e7775e..9a43b95e 100644
--- a/datasette/facets.py
+++ b/datasette/facets.py
@@ -180,7 +180,11 @@ class ColumnFacet(Facet):
                             "name": column,
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(self.request, {"_facet": column}),
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet": column}
+                                    )
+                                ),
                             ),
                         }
                     )
@@ -334,8 +338,10 @@ class ArrayFacet(Facet):
                                 "type": "array",
                                 "toggle_url": self.ds.absolute_url(
                                     self.request,
-                                    path_with_added_args(
-                                        self.request, {"_facet_array": column}
+                                    self.ds.urls.path(
+                                        path_with_added_args(
+                                            self.request, {"_facet_array": column}
+                                        )
                                     ),
                                 ),
                             }
@@ -461,8 +467,10 @@ class DateFacet(Facet):
                             "type": "date",
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(
-                                    self.request, {"_facet_date": column}
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet_date": column}
+                                    )
                                 ),
                             ),
                         }
diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html
index 649f5171..d91a1a57 100644
--- a/datasette/templates/_table.html
+++ b/datasette/templates/_table.html
@@ -9,9 +9,9 @@
                             {{ column.name }}
                         {% else %}
                             {% if column.name == sort %}
-                                {{ column.name }} ▼
+                                {{ column.name }} ▼
                             {% else %}
-                                {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %}
+                                {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %}
                             {% endif %}
                         {% endif %}

    		1hello2world3\xa0{}{i}{i}a{i}b{i}c{i}a,babc1hello\xa01-\xa031ab2\xa0\xa0\xa0\xa0\xa01131ab1world2\xa011,feline1\xa01feline2,canine2\xa02caninehelloHELLOworldWORLD\xa0\xa01<Binary:\xa07\xa0bytes>22<Binary:\xa07\xa0bytes>33\xa03Detroit2Los Angeles4Memnonia1San Francisco2Paranormal1Museum1hello2world3\xa0{}{i}{i}a{i}b{i}c{i}a,babc1hello\xa01-\xa031ab2\xa0\xa0\xa0\xa0\xa01131ab1world2\xa011,feline1\xa01feline2,canine2\xa02caninehelloHELLOworldWORLD\xa0\xa011<Binary:\xa07\xa0bytes>22<Binary:\xa07\xa0bytes>33\xa03Detroit2Los Angeles4Memnonia1San Francisco2Paranormal1Museum
    + Content-Type: application/json + Authorization: Bearer dstok_ + { + "row": { + "column1": "value1", + "column2": "value2" + } + } + +If successful, this will return a ``201`` status code and the newly inserted row, for example: + +.. code-block:: json + + { + "row": { + "id": 1, + "column1": "value1", + "column2": "value2" + } + } From f6ca86987ba9d7d48eccf2cfe0bfc94942003844 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 06:56:11 -0700 Subject: [PATCH 0830/1866] Delete mirror-master-and-main.yml Closes #1865 --- .github/workflows/mirror-master-and-main.yml | 21 -------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/mirror-master-and-main.yml diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml deleted file mode 100644 index 8418df40..00000000 --- a/.github/workflows/mirror-master-and-main.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror "master" and "main" branches -on: - push: - branches: - - master - - main - -jobs: - mirror: - runs-on: ubuntu-latest - steps: - - name: Mirror to "master" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: master - force: false - - name: Mirror to "main" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: main - force: false From 5f6be3c48b661f74198b8fc85361d3ad6657880e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:47:41 -0700 Subject: [PATCH 0831/1866] Better comment handling in SQL regex, refs #1860 --- datasette/utils/__init__.py | 9 +++++---- tests/test_utils.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 977a66d6..5acfb8b4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -208,16 +208,16 @@ class InvalidSql(Exception): # Allow SQL to start with a /* */ or -- comment comment_re = ( # Start of string, then any amount of whitespace - r"^(\s*" + r"^\s*(" + # Comment that starts with -- and ends at a newline r"(?:\-\-.*?\n\s*)" + - # Comment that starts with /* and ends with */ - r"|(?:/\*[\s\S]*?\*/)" + # Comment that starts with /* and ends with */ - but does not have */ in it + r"|(?:\/\*((?!\*\/)[\s\S])*\*\/)" + # Whitespace - r")*\s*" + r"\s*)*\s*" ) allowed_sql_res = [ @@ -228,6 +228,7 @@ allowed_sql_res = [ re.compile(comment_re + r"explain\s+with\b"), re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] + allowed_pragmas = ( "database_list", "foreign_key_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index e89f1e6b..c1589107 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,7 @@ def test_custom_json_encoder(obj, expected): "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", "/* This comment is not valid. select 1", + "/**/\nupdate foo set bar = 1\n/* test */ select 1", ], ) def test_validate_sql_select_bad(bad_sql): From d2ca13b699d441a201c55cb72ff96919d3cd22bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:50:54 -0700 Subject: [PATCH 0832/1866] Add test for /* multi line */ comment, refs #1860 --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c1589107..8b64f865 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -174,6 +174,7 @@ def test_validate_sql_select_bad(bad_sql): " /* comment */\nselect 1", " /* comment */select 1", "/* comment */\n -- another\n /* one more */ select 1", + "/* This comment \n has multiple lines */\nselect 1", ], ) def test_validate_sql_select_good(good_sql): From 918f3561208ee58c44773d30e21bace7d7c7cf3b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 06:56:11 -0700 Subject: [PATCH 0833/1866] Delete mirror-master-and-main.yml Closes #1865 --- .github/workflows/mirror-master-and-main.yml | 21 -------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/mirror-master-and-main.yml diff --git a/.github/workflows/mirror-master-and-main.yml b/.github/workflows/mirror-master-and-main.yml deleted file mode 100644 index 8418df40..00000000 --- a/.github/workflows/mirror-master-and-main.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Mirror "master" and "main" branches -on: - push: - branches: - - master - - main - -jobs: - mirror: - runs-on: ubuntu-latest - steps: - - name: Mirror to "master" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: master - force: false - - name: Mirror to "main" - uses: zofrex/mirror-branch@ea152f124954fa4eb26eea3fe0dbe313a3a08d94 - with: - target-branch: main - force: false From b597bb6b3e7c4b449654bbfa5b01ceff3eb3cb33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:47:41 -0700 Subject: [PATCH 0834/1866] Better comment handling in SQL regex, refs #1860 --- datasette/utils/__init__.py | 9 +++++---- tests/test_utils.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 977a66d6..5acfb8b4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -208,16 +208,16 @@ class InvalidSql(Exception): # Allow SQL to start with a /* */ or -- comment comment_re = ( # Start of string, then any amount of whitespace - r"^(\s*" + r"^\s*(" + # Comment that starts with -- and ends at a newline r"(?:\-\-.*?\n\s*)" + - # Comment that starts with /* and ends with */ - r"|(?:/\*[\s\S]*?\*/)" + # Comment that starts with /* and ends with */ - but does not have */ in it + r"|(?:\/\*((?!\*\/)[\s\S])*\*\/)" + # Whitespace - r")*\s*" + r"\s*)*\s*" ) allowed_sql_res = [ @@ -228,6 +228,7 @@ allowed_sql_res = [ re.compile(comment_re + r"explain\s+with\b"), re.compile(comment_re + r"explain\s+query\s+plan\s+with\b"), ] + allowed_pragmas = ( "database_list", "foreign_key_list", diff --git a/tests/test_utils.py b/tests/test_utils.py index e89f1e6b..c1589107 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,7 @@ def test_custom_json_encoder(obj, expected): "PRAGMA case_sensitive_like = true", "SELECT * FROM pragma_not_on_allow_list('idx52')", "/* This comment is not valid. select 1", + "/**/\nupdate foo set bar = 1\n/* test */ select 1", ], ) def test_validate_sql_select_bad(bad_sql): From 6958e21b5c2012adf5655d2512cb4106490d10f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 11:50:54 -0700 Subject: [PATCH 0835/1866] Add test for /* multi line */ comment, refs #1860 --- tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index c1589107..8b64f865 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -174,6 +174,7 @@ def test_validate_sql_select_bad(bad_sql): " /* comment */\nselect 1", " /* comment */select 1", "/* comment */\n -- another\n /* one more */ select 1", + "/* This comment \n has multiple lines */\nselect 1", ], ) def test_validate_sql_select_good(good_sql): From a51608090b5ee37593078f71d18b33767ef3af79 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 12:06:18 -0700 Subject: [PATCH 0836/1866] Slight tweak to insert row API design, refs #1851 https://github.com/simonw/datasette/issues/1851#issuecomment-1292997608 --- datasette/views/table.py | 10 +++++----- docs/json_api.rst | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 74d1c532..056b7b04 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -131,11 +131,11 @@ class TableView(DataView): # TODO: handle form-encoded data raise BadRequest("Must send JSON data") data = json.loads(await request.post_body()) - if "row" not in data: - raise BadRequest('Must send "row" data') - row = data["row"] + if "insert" not in data: + raise BadRequest('Must send a "insert" key containing a dictionary') + row = data["insert"] if not isinstance(row, dict): - raise BadRequest("row must be a dictionary") + raise BadRequest("insert must be a dictionary") # Verify all columns exist columns = await db.table_columns(table_name) pks = await db.primary_keys(table_name) @@ -165,7 +165,7 @@ class TableView(DataView): ).first() return Response.json( { - "row": dict(new_row), + "inserted_row": dict(new_row), }, status=201, ) diff --git a/docs/json_api.rst b/docs/json_api.rst index b339a738..2ed8a354 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -476,7 +476,7 @@ This requires the :ref:`permissions_insert_row` permission. Content-Type: application/json Authorization: Bearer dstok_ { - "row": { + "insert": { "column1": "value1", "column2": "value2" } @@ -487,7 +487,7 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "row": { + "inserted_row": { "id": 1, "column1": "value1", "column2": "value2" From a2a5dff709c6f1676ac30b5e734c2763002562cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 12:08:26 -0700 Subject: [PATCH 0837/1866] Missing tests for insert row API, refs #1851 --- tests/test_api_write.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/test_api_write.py diff --git a/tests/test_api_write.py b/tests/test_api_write.py new file mode 100644 index 00000000..86c221d0 --- /dev/null +++ b/tests/test_api_write.py @@ -0,0 +1,38 @@ +from datasette.app import Datasette +from datasette.utils import sqlite3 +import pytest +import time + + +@pytest.fixture +def ds_write(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute("create table docs (id integer primary key, title text, score float)") + ds = Datasette([db_path]) + yield ds + db.close() + + +@pytest.mark.asyncio +async def test_write_row(ds_write): + token = "dstok_{}".format( + ds_write.sign( + {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" + ) + ) + response = await ds_write.client.post( + "/data/docs", + json={"insert": {"title": "Test", "score": 1.0}}, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + expected_row = {"id": 1, "title": "Test", "score": 1.0} + assert response.status_code == 201 + assert response.json()["inserted_row"] == expected_row + rows = (await ds_write.get_database("data").execute("select * from docs")).rows + assert dict(rows[0]) == expected_row From 6e788b49edf4f842c0817f006eb9d865778eea5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:17:18 -0700 Subject: [PATCH 0838/1866] New URL design /db/table/-/insert, refs #1851 --- datasette/app.py | 6 +++- datasette/views/table.py | 69 +++++++++++++++++++++++++++++++++++++++- docs/json_api.rst | 18 ++++++----- tests/test_api_write.py | 6 ++-- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 894d7f0f..8bc5fe36 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -39,7 +39,7 @@ from .views.special import ( PermissionsDebugView, MessagesDebugView, ) -from .views.table import TableView +from .views.table import TableView, TableInsertView from .views.row import RowView from .renderer import json_renderer from .url_builder import Urls @@ -1262,6 +1262,10 @@ class Datasette: RowView.as_view(self), r"/(?P[^\/\.]+)/(?P
    [^/]+?)/(?P[^/]+?)(\.(?P\w+))?$", ) + add_route( + TableInsertView.as_view(self), + r"/(?P[^\/\.]+)/(?P
    [^\/\.]+)/-/insert$", + ) return [ # Compile any strings to regular expressions ((re.compile(pattern) if isinstance(pattern, str) else pattern), view) diff --git a/datasette/views/table.py b/datasette/views/table.py index 056b7b04..be3d4f93 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -30,7 +30,7 @@ from datasette.utils import ( ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters -from .base import DataView, DatasetteError, ureg +from .base import BaseView, DataView, DatasetteError, ureg from .database import QueryView LINK_WITH_LABEL = ( @@ -1077,3 +1077,70 @@ async def display_columns_and_rows( } columns = [first_column] + columns return columns, cell_rows + + +class TableInsertView(BaseView): + name = "table-insert" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + database_route = tilde_decode(request.url_vars["database"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database_name = db.name + table_name = tilde_decode(request.url_vars["table"]) + # Table must exist (may handle table creation in the future) + db = self.ds.get_database(database_name) + if not await db.table_exists(table_name): + raise NotFound("Table not found: {}".format(table_name)) + # Must have insert-row permission + if not await self.ds.permission_allowed( + request.actor, "insert-row", resource=(database_name, table_name) + ): + raise Forbidden("Permission denied") + if request.headers.get("content-type") != "application/json": + # TODO: handle form-encoded data + raise BadRequest("Must send JSON data") + data = json.loads(await request.post_body()) + if "row" not in data: + raise BadRequest('Must send a "row" key containing a dictionary') + row = data["row"] + if not isinstance(row, dict): + raise BadRequest("row must be a dictionary") + # Verify all columns exist + columns = await db.table_columns(table_name) + pks = await db.primary_keys(table_name) + for key in row: + if key not in columns: + raise BadRequest("Column not found: {}".format(key)) + if key in pks: + raise BadRequest( + "Cannot insert into primary key column: {}".format(key) + ) + # Perform the insert + sql = "INSERT INTO [{table}] ({columns}) VALUES ({values})".format( + table=escape_sqlite(table_name), + columns=", ".join(escape_sqlite(c) for c in row), + values=", ".join("?" for c in row), + ) + cursor = await db.execute_write(sql, list(row.values())) + # Return the new row + rowid = cursor.lastrowid + new_row = ( + await db.execute( + "SELECT * FROM [{table}] WHERE rowid = ?".format( + table=escape_sqlite(table_name) + ), + [rowid], + ) + ).first() + return Response.json( + { + "inserted": [dict(new_row)], + }, + status=201, + ) diff --git a/docs/json_api.rst b/docs/json_api.rst index 2ed8a354..4a7961f2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -463,7 +463,7 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. -.. _json_api_write_insert_row: +.. _TableInsertView: Inserting a single row ~~~~~~~~~~~~~~~~~~~~~~ @@ -472,11 +472,11 @@ This requires the :ref:`permissions_insert_row` permission. :: - POST //
    + POST //
    /-/insert Content-Type: application/json Authorization: Bearer dstok_ { - "insert": { + "row": { "column1": "value1", "column2": "value2" } @@ -487,9 +487,11 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "inserted_row": { - "id": 1, - "column1": "value1", - "column2": "value2" - } + "inserted": [ + { + "id": 1, + "column1": "value1", + "column2": "value2" + } + ] } diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 86c221d0..e8222e43 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -24,8 +24,8 @@ async def test_write_row(ds_write): ) ) response = await ds_write.client.post( - "/data/docs", - json={"insert": {"title": "Test", "score": 1.0}}, + "/data/docs/-/insert", + json={"row": {"title": "Test", "score": 1.0}}, headers={ "Authorization": "Bearer {}".format(token), "Content-Type": "application/json", @@ -33,6 +33,6 @@ async def test_write_row(ds_write): ) expected_row = {"id": 1, "title": "Test", "score": 1.0} assert response.status_code == 201 - assert response.json()["inserted_row"] == expected_row + assert response.json()["inserted"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row From b912d92b651c4f0b5137da924d135654511f0fe0 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 27 Oct 2022 16:51:20 -0400 Subject: [PATCH 0839/1866] Make hash and size a lazy property (#1837) * use inspect data for hash and file size * make hash and cached_size lazy properties * move hash property near size --- datasette/database.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d75bd70c..af1df0a8 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -39,7 +39,7 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.hash = None + self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None @@ -47,14 +47,6 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - if not self.is_mutable and not self.is_memory: - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.hash = self.ds.inspect_data[self.name]["hash"] - self.cached_size = self.ds.inspect_data[self.name]["size"] - else: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size @property def cached_table_counts(self): @@ -266,14 +258,34 @@ class Database: results = await self.execute_fn(sql_operation_in_thread) return results + @property + def hash(self): + if self.cached_hash is not None: + return self.cached_hash + elif self.is_mutable or self.is_memory: + return None + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_hash = self.ds.inspect_data[self.name]["hash"] + return self.cached_hash + else: + p = Path(self.path) + self.cached_hash = inspect_hash(p) + return self.cached_hash + @property def size(self): - if self.is_memory: - return 0 if self.cached_size is not None: return self.cached_size - else: + elif self.is_memory: + return 0 + elif self.is_mutable: return Path(self.path).stat().st_size + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_size = self.ds.inspect_data[self.name]["size"] + return self.cached_size + else: + self.cached_size = Path(self.path).stat().st_size + return self.cached_size async def table_counts(self, limit=10): if not self.is_mutable and self.cached_table_counts is not None: From 2c36e45447494cd7505440943367e29ec57c8e72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 13:51:45 -0700 Subject: [PATCH 0840/1866] Bump black from 22.8.0 to 22.10.0 (#1839) Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe258adb..625557ae 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.8.0", + "black==22.10.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From e5e0459a0b60608cb5e9ff83f6b41f59e6cafdfd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:58:00 -0700 Subject: [PATCH 0841/1866] Release notes for 0.63, refs #1869 --- docs/changelog.rst | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2255dcce..01957e4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,36 +4,42 @@ Changelog ========= -.. _v0_63a1: +.. _v0_63: -0.63a1 (2022-10-23) -------------------- +0.63 (2022-10-27) +----------------- +Features +~~~~~~~~ + +- Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`) +- The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) +- SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks, Charles Nepote. (:issue:`1860`) - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) -- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) - The :ref:`inspect data ` mechanism is now used to speed up server startup - thanks, Forest Gregg. (:issue:`1834`) - In :ref:`config_dir` databases with filenames ending in ``.sqlite`` or ``.sqlite3`` are now automatically added to the Datasette instance. (:issue:`1646`) - Breadcrumb navigation display now respects the current user's permissions. (:issue:`1831`) -- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) -- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - -.. _v0_63a0: - -0.63a0 (2022-09-26) -------------------- +Plugin hooks and internals +~~~~~~~~~~~~~~~~~~~~~~~~~~ - The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) -- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) -- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. -- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) -- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) - ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) -- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) -- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - Datasette no longer enforces upper bounds on its dependencies. (:issue:`1800`) -- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) -- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + +Documentation +~~~~~~~~~~~~~ + +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) .. _v0_62: From bf00b0b59b6692bdec597ac9db4e0b497c5a47b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 15:11:26 -0700 Subject: [PATCH 0842/1866] Release 0.63 Refs #1646, #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816, #1819, #1825, #1829, #1831, #1834, #1844, #1853, #1860 Closes #1869 --- datasette/version.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index eb36da45..ac012640 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63a1" +__version__ = "0.63" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01957e4f..f573afb3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog 0.63 (2022-10-27) ----------------- +See `Datasette 0.63: The annotated release notes `__ for more background on the changes in this release. + Features ~~~~~~~~ From 2ea60e12d90b7cec03ebab728854d3ec4d553f54 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 27 Oct 2022 16:51:20 -0400 Subject: [PATCH 0843/1866] Make hash and size a lazy property (#1837) * use inspect data for hash and file size * make hash and cached_size lazy properties * move hash property near size --- datasette/database.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d75bd70c..af1df0a8 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -39,7 +39,7 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.hash = None + self.cached_hash = None self.cached_size = None self._cached_table_counts = None self._write_thread = None @@ -47,14 +47,6 @@ class Database: # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None - if not self.is_mutable and not self.is_memory: - if self.ds.inspect_data and self.ds.inspect_data.get(self.name): - self.hash = self.ds.inspect_data[self.name]["hash"] - self.cached_size = self.ds.inspect_data[self.name]["size"] - else: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size @property def cached_table_counts(self): @@ -266,14 +258,34 @@ class Database: results = await self.execute_fn(sql_operation_in_thread) return results + @property + def hash(self): + if self.cached_hash is not None: + return self.cached_hash + elif self.is_mutable or self.is_memory: + return None + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_hash = self.ds.inspect_data[self.name]["hash"] + return self.cached_hash + else: + p = Path(self.path) + self.cached_hash = inspect_hash(p) + return self.cached_hash + @property def size(self): - if self.is_memory: - return 0 if self.cached_size is not None: return self.cached_size - else: + elif self.is_memory: + return 0 + elif self.is_mutable: return Path(self.path).stat().st_size + elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.cached_size = self.ds.inspect_data[self.name]["size"] + return self.cached_size + else: + self.cached_size = Path(self.path).stat().st_size + return self.cached_size async def table_counts(self, limit=10): if not self.is_mutable and self.cached_table_counts is not None: From 641bc4453b5ef1dff0b2fc7dfad0b692be7aa61c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Oct 2022 13:51:45 -0700 Subject: [PATCH 0844/1866] Bump black from 22.8.0 to 22.10.0 (#1839) Bumps [black](https://github.com/psf/black) from 22.8.0 to 22.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.8.0...22.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe258adb..625557ae 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.8.0", + "black==22.10.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From 26af9b9c4a6c62ee15870caa1c7bc455165d3b11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 13:58:00 -0700 Subject: [PATCH 0845/1866] Release notes for 0.63, refs #1869 --- docs/changelog.rst | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2255dcce..01957e4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,36 +4,42 @@ Changelog ========= -.. _v0_63a1: +.. _v0_63: -0.63a1 (2022-10-23) -------------------- +0.63 (2022-10-27) +----------------- +Features +~~~~~~~~ + +- Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`) +- The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) +- SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks, Charles Nepote. (:issue:`1860`) - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) -- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) - The :ref:`inspect data ` mechanism is now used to speed up server startup - thanks, Forest Gregg. (:issue:`1834`) - In :ref:`config_dir` databases with filenames ending in ``.sqlite`` or ``.sqlite3`` are now automatically added to the Datasette instance. (:issue:`1646`) - Breadcrumb navigation display now respects the current user's permissions. (:issue:`1831`) -- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) -- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - -.. _v0_63a0: - -0.63a0 (2022-09-26) -------------------- +Plugin hooks and internals +~~~~~~~~~~~~~~~~~~~~~~~~~~ - The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) -- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) -- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. -- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) -- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) - ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) -- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) -- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- The :ref:`datasette.check_visibility() ` method now accepts an optional ``permissions=`` list, allowing it to take multiple permissions into account at once when deciding if something should be shown as public or private. This has been used to correctly display padlock icons in more places in the Datasette interface. (:issue:`1829`) - Datasette no longer enforces upper bounds on its dependencies. (:issue:`1800`) -- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) -- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + +Documentation +~~~~~~~~~~~~~ + +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Screenshots in the documentation are now maintained using `shot-scraper `__, as described in `Automating screenshots for the Datasette documentation using shot-scraper `__. (:issue:`1844`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 `__) .. _v0_62: From 61171f01549549e5fb25c72b13280d941d96dbf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 15:11:26 -0700 Subject: [PATCH 0846/1866] Release 0.63 Refs #1646, #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816, #1819, #1825, #1829, #1831, #1834, #1844, #1853, #1860 Closes #1869 --- datasette/version.py | 2 +- docs/changelog.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index eb36da45..ac012640 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.63a1" +__version__ = "0.63" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01957e4f..f573afb3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog 0.63 (2022-10-27) ----------------- +See `Datasette 0.63: The annotated release notes `__ for more background on the changes in this release. + Features ~~~~~~~~ From c9b5f5d598e7f85cd3e1ce020351a27da334408b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 27 Oct 2022 17:58:36 -0700 Subject: [PATCH 0847/1866] Depend on sqlite-utils>=3.30 Decided to use the most recent version in case I decide later to use the flatten() utility function. Refs #1850 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 625557ae..99e2a4ad 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", + "sqlite-utils>=3.30", ], entry_points=""" [console_scripts] From c35859ae3df163406f1a1895ccf9803e933b2d8e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 29 Oct 2022 23:03:45 -0700 Subject: [PATCH 0848/1866] API for bulk inserts, closes #1866 --- datasette/app.py | 5 ++ datasette/views/table.py | 136 +++++++++++++++++++++---------- docs/cli-reference.rst | 2 + docs/json_api.rst | 48 ++++++++++- docs/settings.rst | 11 +++ tests/test_api.py | 1 + tests/test_api_write.py | 168 +++++++++++++++++++++++++++++++++++++-- 7 files changed, 320 insertions(+), 51 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8bc5fe36..f80d3792 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -99,6 +99,11 @@ SETTINGS = ( 1000, "Maximum rows that can be returned from a table or custom query", ), + Setting( + "max_insert_rows", + 100, + "Maximum rows that can be inserted at a time using the bulk insert API", + ), Setting( "num_sql_threads", 3, diff --git a/datasette/views/table.py b/datasette/views/table.py index be3d4f93..fd203036 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -30,6 +30,7 @@ from datasette.utils import ( ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters +import sqlite_utils from .base import BaseView, DataView, DatasetteError, ureg from .database import QueryView @@ -1085,62 +1086,109 @@ class TableInsertView(BaseView): def __init__(self, datasette): self.ds = datasette + async def _validate_data(self, request, db, table_name): + errors = [] + + def _errors(errors): + return None, errors, {} + + if request.headers.get("content-type") != "application/json": + # TODO: handle form-encoded data + return _errors(["Invalid content-type, must be application/json"]) + body = await request.post_body() + try: + data = json.loads(body) + except json.JSONDecodeError as e: + return _errors(["Invalid JSON: {}".format(e)]) + if not isinstance(data, dict): + return _errors(["JSON must be a dictionary"]) + keys = data.keys() + # keys must contain "row" or "rows" + if "row" not in keys and "rows" not in keys: + return _errors(['JSON must have one or other of "row" or "rows"']) + rows = [] + if "row" in keys: + if "rows" in keys: + return _errors(['Cannot use "row" and "rows" at the same time']) + row = data["row"] + if not isinstance(row, dict): + return _errors(['"row" must be a dictionary']) + rows = [row] + data["return_rows"] = True + else: + rows = data["rows"] + if not isinstance(rows, list): + return _errors(['"rows" must be a list']) + for row in rows: + if not isinstance(row, dict): + return _errors(['"rows" must be a list of dictionaries']) + # Does this exceed max_insert_rows? + max_insert_rows = self.ds.setting("max_insert_rows") + if len(rows) > max_insert_rows: + return _errors( + ["Too many rows, maximum allowed is {}".format(max_insert_rows)] + ) + # Validate columns of each row + columns = await db.table_columns(table_name) + # TODO: There are cases where pks are OK, if not using auto-incrementing pk + pks = await db.primary_keys(table_name) + allowed_columns = set(columns) - set(pks) + for i, row in enumerate(rows): + invalid_columns = set(row.keys()) - allowed_columns + if invalid_columns: + errors.append( + "Row {} has invalid columns: {}".format( + i, ", ".join(sorted(invalid_columns)) + ) + ) + if errors: + return _errors(errors) + extra = {key: data[key] for key in data if key not in ("rows", "row")} + return rows, errors, extra + async def post(self, request): + def _error(messages, status=400): + return Response.json({"ok": False, "errors": messages}, status=status) + database_route = tilde_decode(request.url_vars["database"]) try: db = self.ds.get_database(route=database_route) except KeyError: - raise NotFound("Database not found: {}".format(database_route)) + return _error(["Database not found: {}".format(database_route)], 404) database_name = db.name table_name = tilde_decode(request.url_vars["table"]) + # Table must exist (may handle table creation in the future) db = self.ds.get_database(database_name) if not await db.table_exists(table_name): - raise NotFound("Table not found: {}".format(table_name)) + return _error(["Table not found: {}".format(table_name)], 404) # Must have insert-row permission if not await self.ds.permission_allowed( request.actor, "insert-row", resource=(database_name, table_name) ): - raise Forbidden("Permission denied") - if request.headers.get("content-type") != "application/json": - # TODO: handle form-encoded data - raise BadRequest("Must send JSON data") - data = json.loads(await request.post_body()) - if "row" not in data: - raise BadRequest('Must send a "row" key containing a dictionary') - row = data["row"] - if not isinstance(row, dict): - raise BadRequest("row must be a dictionary") - # Verify all columns exist - columns = await db.table_columns(table_name) - pks = await db.primary_keys(table_name) - for key in row: - if key not in columns: - raise BadRequest("Column not found: {}".format(key)) - if key in pks: - raise BadRequest( - "Cannot insert into primary key column: {}".format(key) + return _error(["Permission denied"], 403) + rows, errors, extra = await self._validate_data(request, db, table_name) + if errors: + return _error(errors, 400) + + should_return = bool(extra.get("return_rows", False)) + # Insert rows + def insert_rows(conn): + table = sqlite_utils.Database(conn)[table_name] + if should_return: + rowids = [] + for row in rows: + rowids.append(table.insert(row).last_rowid) + return list( + table.rows_where( + "rowid in ({})".format(",".join("?" for _ in rowids)), rowids + ) ) - # Perform the insert - sql = "INSERT INTO [{table}] ({columns}) VALUES ({values})".format( - table=escape_sqlite(table_name), - columns=", ".join(escape_sqlite(c) for c in row), - values=", ".join("?" for c in row), - ) - cursor = await db.execute_write(sql, list(row.values())) - # Return the new row - rowid = cursor.lastrowid - new_row = ( - await db.execute( - "SELECT * FROM [{table}] WHERE rowid = ?".format( - table=escape_sqlite(table_name) - ), - [rowid], - ) - ).first() - return Response.json( - { - "inserted": [dict(new_row)], - }, - status=201, - ) + else: + table.insert_all(rows) + + rows = await db.execute_write_fn(insert_rows) + result = {"ok": True} + if should_return: + result["inserted"] = rows + return Response.json(result, status=201) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 56156568..649a3dcd 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -213,6 +213,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam (default=100) max_returned_rows Maximum rows that can be returned from a table or custom query (default=1000) + max_insert_rows Maximum rows that can be inserted at a time using + the bulk insert API (default=1000) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/json_api.rst b/docs/json_api.rst index 4a7961f2..01558c23 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -465,11 +465,13 @@ Datasette provides a write API for JSON data. This is a POST-only API that requi .. _TableInsertView: -Inserting a single row -~~~~~~~~~~~~~~~~~~~~~~ +Inserting rows +~~~~~~~~~~~~~~ This requires the :ref:`permissions_insert_row` permission. +A single row can be inserted using the ``"row"`` key: + :: POST //
    /-/insert @@ -495,3 +497,45 @@ If successful, this will return a ``201`` status code and the newly inserted row } ] } + +To insert multiple rows at a time, use the same API method but send a list of dictionaries as the ``"rows"`` key: + +:: + + POST //
    /-/insert + Content-Type: application/json + Authorization: Bearer dstok_ + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ] + } + +If successful, this will return a ``201`` status code and an empty ``{}`` response body. + +To return the newly inserted rows, add the ``"return_rows": true`` key to the request body: + +.. code-block:: json + + { + "rows": [ + { + "column1": "value1", + "column2": "value2" + }, + { + "column1": "value3", + "column2": "value4" + } + ], + "return_rows": true + } + +This will return the same ``"inserted"`` key as the single row example above. There is a small performance penalty for using this option. diff --git a/docs/settings.rst b/docs/settings.rst index a990c78c..b86b18bd 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -96,6 +96,17 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_returned_rows 2000 +.. _setting_max_insert_rows: + +max_insert_rows +~~~~~~~~~~~~~~~ + +Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100. + +You can increase or decrease this limit like so:: + + datasette mydatabase.db --setting max_insert_rows 1000 + .. _setting_num_sql_threads: num_sql_threads diff --git a/tests/test_api.py b/tests/test_api.py index fc171421..ebd675b9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -804,6 +804,7 @@ def test_settings_json(app_client): "facet_suggest_time_limit_ms": 50, "facet_time_limit_ms": 200, "max_returned_rows": 100, + "max_insert_rows": 100, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index e8222e43..4a5a58aa 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -18,11 +18,7 @@ def ds_write(tmp_path_factory): @pytest.mark.asyncio async def test_write_row(ds_write): - token = "dstok_{}".format( - ds_write.sign( - {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" - ) - ) + token = write_token(ds_write) response = await ds_write.client.post( "/data/docs/-/insert", json={"row": {"title": "Test", "score": 1.0}}, @@ -36,3 +32,165 @@ async def test_write_row(ds_write): assert response.json()["inserted"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row + + +@pytest.mark.asyncio +@pytest.mark.parametrize("return_rows", (True, False)) +async def test_write_rows(ds_write, return_rows): + token = write_token(ds_write) + data = {"rows": [{"title": "Test {}".format(i), "score": 1.0} for i in range(20)]} + if return_rows: + data["return_rows"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert len(actual_rows) == 20 + assert actual_rows == [ + {"id": i + 1, "title": "Test {}".format(i), "score": 1.0} for i in range(20) + ] + assert response.json()["ok"] is True + if return_rows: + assert response.json()["inserted"] == actual_rows + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,input,special_case,expected_status,expected_errors", + ( + ( + "/data2/docs/-/insert", + {}, + None, + 404, + ["Database not found: data2"], + ), + ( + "/data/docs2/-/insert", + {}, + None, + 404, + ["Table not found: docs2"], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(10)]}, + "bad_token", + 403, + ["Permission denied"], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + "/data/docs/-/insert", + {}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + "/data/docs/-/insert", + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + "/data/docs/-/insert", + {"row": "blah"}, + None, + 400, + ['"row" must be a dictionary'], + ), + ( + "/data/docs/-/insert", + {"blah": "blah"}, + None, + 400, + ['JSON must have one or other of "row" or "rows"'], + ), + ( + "/data/docs/-/insert", + {"rows": "blah"}, + None, + 400, + ['"rows" must be a list'], + ), + ( + "/data/docs/-/insert", + {"rows": ["blah"]}, + None, + 400, + ['"rows" must be a list of dictionaries'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"} for i in range(101)]}, + None, + 400, + ["Too many rows, maximum allowed is 100"], + ), + # Validate columns of each row + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test", "bad": 1, "worse": 2} for i in range(2)]}, + None, + 400, + [ + "Row 0 has invalid columns: bad, worse", + "Row 1 has invalid columns: bad, worse", + ], + ), + ), +) +async def test_write_row_errors( + ds_write, path, input, special_case, expected_status, expected_errors +): + token = write_token(ds_write) + if special_case == "bad_token": + token += "bad" + kwargs = dict( + json=input, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "text/plain" + if special_case == "invalid_content_type" + else "application/json", + }, + ) + if special_case == "invalid_json": + del kwargs["json"] + kwargs["content"] = "{bad json" + response = await ds_write.client.post( + path, + **kwargs, + ) + assert response.status_code == expected_status + assert response.json()["ok"] is False + assert response.json()["errors"] == expected_errors + + +def write_token(ds): + return "dstok_{}".format( + ds.sign( + {"a": "root", "token": "dstok", "t": int(time.time())}, namespace="token" + ) + ) From f6bf2d8045cc239fe34357342bff1440561c8909 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 29 Oct 2022 23:20:11 -0700 Subject: [PATCH 0849/1866] Initial prototype of API explorer at /-/api, refs #1871 --- datasette/app.py | 5 ++ datasette/templates/api_explorer.html | 73 +++++++++++++++++++++++++++ datasette/views/special.py | 8 +++ tests/test_docs.py | 2 +- 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 datasette/templates/api_explorer.html diff --git a/datasette/app.py b/datasette/app.py index f80d3792..c3d802a4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,6 +33,7 @@ from .views.special import ( JsonDataView, PatternPortfolioView, AuthTokenView, + ApiExplorerView, CreateTokenView, LogoutView, AllowDebugView, @@ -1235,6 +1236,10 @@ class Datasette: CreateTokenView.as_view(self), r"/-/create-token$", ) + add_route( + ApiExplorerView.as_view(self), + r"/-/api$", + ) add_route( LogoutView.as_view(self), r"/-/logout$", diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html new file mode 100644 index 00000000..034bee60 --- /dev/null +++ b/datasette/templates/api_explorer.html @@ -0,0 +1,73 @@ +{% extends "base.html" %} + +{% block title %}API Explorer{% endblock %} + +{% block content %} + +

    API Explorer

    + +

    Use this tool to try out the Datasette write API.

    + +{% if errors %} + {% for error in errors %} +

    {{ error }}

    + {% endfor %} +{% endif %} + + +
    + + +
    +
    + + +
    +
    + +
    +

    + + + + +{% endblock %} diff --git a/datasette/views/special.py b/datasette/views/special.py index b754a2f0..9922a621 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -235,3 +235,11 @@ class CreateTokenView(BaseView): "token_bits": token_bits, }, ) + + +class ApiExplorerView(BaseView): + name = "api_explorer" + has_json_alternate = False + + async def get(self, request): + return await self.render(["api_explorer.html"], request) diff --git a/tests/test_docs.py b/tests/test_docs.py index cd5a6c13..e9b813fe 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -62,7 +62,7 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView")) + view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) return view_labels From 9eb9ffae3ddd4e8ff0b713bf6fd6a0afed3368d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Oct 2022 13:09:55 -0700 Subject: [PATCH 0850/1866] Drop API token requirement from API explorer, refs #1871 --- datasette/default_permissions.py | 9 +++++++++ datasette/templates/api_explorer.html | 13 ++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 87684e2a..151ba2b5 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -131,3 +131,12 @@ def register_commands(cli): if debug: click.echo("\nDecoded:\n") click.echo(json.dumps(ds.unsign(token, namespace="token"), indent=2)) + + +@hookimpl +def skip_csrf(scope): + # Skip CSRF check for requests with content-type: application/json + if scope["type"] == "http": + headers = scope.get("headers") or {} + if dict(headers).get(b"content-type") == b"application/json": + return True diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 034bee60..01b182d8 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -15,16 +15,13 @@ {% endif %}
    -
    - - -
    - +
    -
    - +
    + +

    @@ -46,7 +43,6 @@ form.addEventListener("submit", (ev) => { var formData = new FormData(form); var json = formData.get('json'); var path = formData.get('path'); - var token = formData.get('token'); // Validate JSON try { var data = JSON.parse(json); @@ -60,7 +56,6 @@ form.addEventListener("submit", (ev) => { body: json, headers: { 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` } }).then(r => r.json()).then(r => { alert(JSON.stringify(r, null, 2)); From fedbfcc36873366143195d8fe124e1859bf88346 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 30 Oct 2022 14:49:07 -0700 Subject: [PATCH 0851/1866] Neater display of output and errors in API explorer, refs #1871 --- datasette/templates/api_explorer.html | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 01b182d8..38fdb7bc 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -26,6 +26,12 @@

    + + """.format( escape(ex.sql) ) diff --git a/tests/test_api.py b/tests/test_api.py index ad74d16e..4027a7a5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -662,7 +662,11 @@ def test_sql_time_limit(app_client_shorter_time_limit): "

    SQL query took too long. The time limit is controlled by the\n" 'sql_time_limit_ms\n' "configuration option.

    \n" - "
    select sleep(0.5)
    " + '\n' + "" ), "status": 400, "title": "SQL Interrupted", diff --git a/tests/test_html.py b/tests/test_html.py index 4b394199..7cfe9d90 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -172,7 +172,7 @@ def test_sql_time_limit(app_client_shorter_time_limit): """ sql_time_limit_ms """.strip(), - "
    select sleep(0.5)
    ", + '', ] for expected_html_fragment in expected_html_fragments: assert expected_html_fragment in response.text From 93a02281dad2f23da84210f6ae9c63777ad8af5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 10:22:26 -0700 Subject: [PATCH 0856/1866] Show interrupted query in resizing textarea, closes #1876 --- datasette/views/base.py | 6 +++++- tests/test_api.py | 6 +++++- tests/test_html.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 67aa3a42..6b01fdd2 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -378,7 +378,11 @@ class DataView(BaseView):

    SQL query took too long. The time limit is controlled by the sql_time_limit_ms configuration option.

    - 
    {}
    + + """.format( escape(ex.sql) ) diff --git a/tests/test_api.py b/tests/test_api.py index ebd675b9..de0223e2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -662,7 +662,11 @@ def test_sql_time_limit(app_client_shorter_time_limit): "

    SQL query took too long. The time limit is controlled by the\n" 'sql_time_limit_ms\n' "configuration option.

    \n" - "
    select sleep(0.5)
    " + '\n' + "" ), "status": 400, "title": "SQL Interrupted", diff --git a/tests/test_html.py b/tests/test_html.py index 4b394199..7cfe9d90 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -172,7 +172,7 @@ def test_sql_time_limit(app_client_shorter_time_limit): """ sql_time_limit_ms """.strip(), - "
    select sleep(0.5)
    ", + '', ] for expected_html_fragment in expected_html_fragments: assert expected_html_fragment in response.text From 9bec7c38eb93cde5afb16df9bdd96aea2a5b0459 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 11:07:59 -0700 Subject: [PATCH 0857/1866] ignore and replace options for bulk inserts, refs #1873 Also removed the rule that you cannot include primary keys in the rows you insert. And added validation that catches invalid parameters in the incoming JSON. And renamed "inserted" to "rows" in the returned JSON for return_rows: true --- datasette/views/table.py | 41 ++++++++++++++------ docs/json_api.rst | 4 +- tests/test_api_write.py | 83 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 17 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1e3d566e..7692a4e3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1107,6 +1107,7 @@ class TableInsertView(BaseView): if not isinstance(data, dict): return _errors(["JSON must be a dictionary"]) keys = data.keys() + # keys must contain "row" or "rows" if "row" not in keys and "rows" not in keys: return _errors(['JSON must have one or other of "row" or "rows"']) @@ -1126,19 +1127,31 @@ class TableInsertView(BaseView): for row in rows: if not isinstance(row, dict): return _errors(['"rows" must be a list of dictionaries']) + # Does this exceed max_insert_rows? max_insert_rows = self.ds.setting("max_insert_rows") if len(rows) > max_insert_rows: return _errors( ["Too many rows, maximum allowed is {}".format(max_insert_rows)] ) + + # Validate other parameters + extras = { + key: value for key, value in data.items() if key not in ("row", "rows") + } + valid_extras = {"return_rows", "ignore", "replace"} + invalid_extras = extras.keys() - valid_extras + if invalid_extras: + return _errors( + ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_extras)))] + ) + if extras.get("ignore") and extras.get("replace"): + return _errors(['Cannot use "ignore" and "replace" at the same time']) + # Validate columns of each row - columns = await db.table_columns(table_name) - # TODO: There are cases where pks are OK, if not using auto-incrementing pk - pks = await db.primary_keys(table_name) - allowed_columns = set(columns) - set(pks) + columns = set(await db.table_columns(table_name)) for i, row in enumerate(rows): - invalid_columns = set(row.keys()) - allowed_columns + invalid_columns = set(row.keys()) - columns if invalid_columns: errors.append( "Row {} has invalid columns: {}".format( @@ -1147,8 +1160,7 @@ class TableInsertView(BaseView): ) if errors: return _errors(errors) - extra = {key: data[key] for key in data if key not in ("rows", "row")} - return rows, errors, extra + return rows, errors, extras async def post(self, request): database_route = tilde_decode(request.url_vars["database"]) @@ -1168,18 +1180,23 @@ class TableInsertView(BaseView): request.actor, "insert-row", resource=(database_name, table_name) ): return _error(["Permission denied"], 403) - rows, errors, extra = await self._validate_data(request, db, table_name) + rows, errors, extras = await self._validate_data(request, db, table_name) if errors: return _error(errors, 400) - should_return = bool(extra.get("return_rows", False)) + ignore = extras.get("ignore") + replace = extras.get("replace") + + should_return = bool(extras.get("return_rows", False)) # Insert rows def insert_rows(conn): table = sqlite_utils.Database(conn)[table_name] if should_return: rowids = [] for row in rows: - rowids.append(table.insert(row).last_rowid) + rowids.append( + table.insert(row, ignore=ignore, replace=replace).last_rowid + ) return list( table.rows_where( "rowid in ({})".format(",".join("?" for _ in rowids)), @@ -1187,12 +1204,12 @@ class TableInsertView(BaseView): ) ) else: - table.insert_all(rows) + table.insert_all(rows, ignore=ignore, replace=replace) rows = await db.execute_write_fn(insert_rows) result = {"ok": True} if should_return: - result["inserted"] = rows + result["rows"] = rows return Response.json(result, status=201) diff --git a/docs/json_api.rst b/docs/json_api.rst index da4500ab..34c13211 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -489,7 +489,7 @@ If successful, this will return a ``201`` status code and the newly inserted row .. code-block:: json { - "inserted": [ + "rows": [ { "id": 1, "column1": "value1", @@ -538,7 +538,7 @@ To return the newly inserted rows, add the ``"return_rows": true`` key to the re "return_rows": true } -This will return the same ``"inserted"`` key as the single row example above. There is a small performance penalty for using this option. +This will return the same ``"rows"`` key as the single row example above. There is a small performance penalty for using this option. .. _RowDeleteView: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1cfba104..d0b0f324 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -37,7 +37,7 @@ async def test_write_row(ds_write): ) expected_row = {"id": 1, "title": "Test", "score": 1.0} assert response.status_code == 201 - assert response.json()["inserted"] == [expected_row] + assert response.json()["rows"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row @@ -70,7 +70,7 @@ async def test_write_rows(ds_write, return_rows): ] assert response.json()["ok"] is True if return_rows: - assert response.json()["inserted"] == actual_rows + assert response.json()["rows"] == actual_rows @pytest.mark.asyncio @@ -156,6 +156,27 @@ async def test_write_rows(ds_write, return_rows): 400, ["Too many rows, maximum allowed is 100"], ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "ignore": True, "replace": True}, + None, + 400, + ['Cannot use "ignore" and "replace" at the same time'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "invalid_param": True}, + None, + 400, + ['Invalid parameter: "invalid_param"'], + ), + ( + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "one": True, "two": True}, + None, + 400, + ['Invalid parameter: "one", "two"'], + ), # Validate columns of each row ( "/data/docs/-/insert", @@ -196,6 +217,62 @@ async def test_write_row_errors( assert response.json()["errors"] == expected_errors +@pytest.mark.asyncio +@pytest.mark.parametrize( + "ignore,replace,expected_rows", + ( + ( + True, + False, + [ + {"id": 1, "title": "Exists", "score": None}, + ], + ), + ( + False, + True, + [ + {"id": 1, "title": "One", "score": None}, + ], + ), + ), +) +@pytest.mark.parametrize("should_return", (True, False)) +async def test_insert_ignore_replace( + ds_write, ignore, replace, expected_rows, should_return +): + await ds_write.get_database("data").execute_write( + "insert into docs (id, title) values (1, 'Exists')" + ) + token = write_token(ds_write) + data = {"rows": [{"id": 1, "title": "One"}]} + if ignore: + data["ignore"] = True + if replace: + data["replace"] = True + if should_return: + data["return_rows"] = True + response = await ds_write.client.post( + "/data/docs/-/insert", + json=data, + headers={ + "Authorization": "Bearer {}".format(token), + "Content-Type": "application/json", + }, + ) + assert response.status_code == 201 + actual_rows = [ + dict(r) + for r in ( + await ds_write.get_database("data").execute("select * from docs") + ).rows + ] + assert actual_rows == expected_rows + assert response.json()["ok"] is True + if should_return: + assert response.json()["rows"] == expected_rows + + @pytest.mark.asyncio @pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table", "has_perm")) async def test_delete_row(ds_write, scenario): @@ -217,7 +294,7 @@ async def test_delete_row(ds_write, scenario): }, ) assert insert_response.status_code == 201 - pk = insert_response.json()["inserted"][0]["id"] + pk = insert_response.json()["rows"][0]["id"] path = "/data/{}/{}/-/delete".format( "docs" if scenario != "bad_table" else "bad_table", pk From 497290beaf32e6b779f9683ef15f1c5bc142a41a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 12:59:17 -0700 Subject: [PATCH 0858/1866] Handle database errors in /-/insert, refs #1866, #1873 Also improved API explorer to show HTTP status of response, refs #1871 --- datasette/templates/api_explorer.html | 14 +++++++++----- datasette/views/table.py | 5 ++++- tests/test_api_write.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 38fdb7bc..93bacde3 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -27,7 +27,8 @@ @@ -64,12 +65,15 @@ form.addEventListener("submit", (ev) => { headers: { 'Content-Type': 'application/json', } - }).then(r => r.json()).then(r => { + }).then(r => { + document.getElementById('response-status').textContent = r.status; + return r.json(); + }).then(data => { var errorList = output.querySelector('.errors'); - if (r.errors) { + if (data.errors) { errorList.style.display = 'block'; errorList.innerHTML = ''; - r.errors.forEach(error => { + data.errors.forEach(error => { var li = document.createElement('li'); li.textContent = error; errorList.appendChild(li); @@ -77,7 +81,7 @@ form.addEventListener("submit", (ev) => { } else { errorList.style.display = 'none'; } - output.querySelector('pre').innerText = JSON.stringify(r, null, 2); + output.querySelector('pre').innerText = JSON.stringify(data, null, 2); output.style.display = 'block'; }).catch(err => { alert("Error: " + err); diff --git a/datasette/views/table.py b/datasette/views/table.py index 7692a4e3..61227206 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1206,7 +1206,10 @@ class TableInsertView(BaseView): else: table.insert_all(rows, ignore=ignore, replace=replace) - rows = await db.execute_write_fn(insert_rows) + try: + rows = await db.execute_write_fn(insert_rows) + except Exception as e: + return _error([str(e)]) result = {"ok": True} if should_return: result["rows"] = rows diff --git a/tests/test_api_write.py b/tests/test_api_write.py index d0b0f324..0b567f48 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -156,6 +156,13 @@ async def test_write_rows(ds_write, return_rows): 400, ["Too many rows, maximum allowed is 100"], ), + ( + "/data/docs/-/insert", + {"rows": [{"id": 1, "title": "Test"}]}, + "duplicate_id", + 400, + ["UNIQUE constraint failed: docs.id"], + ), ( "/data/docs/-/insert", {"rows": [{"title": "Test"}], "ignore": True, "replace": True}, @@ -194,6 +201,10 @@ async def test_write_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): token = write_token(ds_write) + if special_case == "duplicate_id": + await ds_write.get_database("data").execute_write( + "insert into docs (id) values (1)" + ) if special_case == "bad_token": token += "bad" kwargs = dict( From 0b166befc0096fca30d71e19608a928d59c331a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 1 Nov 2022 17:31:22 -0700 Subject: [PATCH 0859/1866] API explorer can now do GET, has JSON syntax highlighting Refs #1871 --- .../static/json-format-highlight-1.0.1.js | 43 +++++++++++ datasette/templates/api_explorer.html | 77 +++++++++++++++---- 2 files changed, 103 insertions(+), 17 deletions(-) create mode 100644 datasette/static/json-format-highlight-1.0.1.js diff --git a/datasette/static/json-format-highlight-1.0.1.js b/datasette/static/json-format-highlight-1.0.1.js new file mode 100644 index 00000000..e87c76e1 --- /dev/null +++ b/datasette/static/json-format-highlight-1.0.1.js @@ -0,0 +1,43 @@ +/* +https://github.com/luyilin/json-format-highlight +From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js +MIT Licensed +*/ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : + typeof define === 'function' && define.amd ? define(factory) : + (global.jsonFormatHighlight = factory()); +}(this, (function () { 'use strict'; + +var defaultColors = { + keyColor: 'dimgray', + numberColor: 'lightskyblue', + stringColor: 'lightcoral', + trueColor: 'lightseagreen', + falseColor: '#f66578', + nullColor: 'cornflowerblue' +}; + +function index (json, colorOptions) { + if ( colorOptions === void 0 ) colorOptions = {}; + + if (!json) { return; } + if (typeof json !== 'string') { + json = JSON.stringify(json, null, 2); + } + var colors = Object.assign({}, defaultColors, colorOptions); + json = json.replace(/&/g, '&').replace(//g, '>'); + return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g, function (match) { + var color = colors.numberColor; + if (/^"/.test(match)) { + color = /:$/.test(match) ? colors.keyColor : colors.stringColor; + } else { + color = /true/.test(match) ? colors.trueColor : /false/.test(match) ? colors.falseColor : /null/.test(match) ? colors.nullColor : color; + } + return ("" + match + ""); + }); +} + +return index; + +}))); diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 93bacde3..de5337e3 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -2,6 +2,10 @@ {% block title %}API Explorer{% endblock %} +{% block extra_head %} + +{% endblock %} + {% block content %}

    API Explorer

    @@ -14,17 +18,30 @@ {% endfor %} {% endif %} -
    -
    - - -
    -
    - - -
    -

    - +
    + GET +
    +
    + + + +
    + +
    +
    + POST +
    +
    + + +
    +
    + + +
    +

    + +
    {% else %} - {% if not canned_write and not error %} + {% if not canned_query_write and not error %}

    0 results

    {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0770a380..658c35e6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,3 @@ -from asyncinject import Registry from dataclasses import dataclass, field from typing import Callable from urllib.parse import parse_qsl, urlencode @@ -33,7 +32,7 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, DataView, View, _error, stream_csv +from .base import BaseView, DatasetteError, View, _error, stream_csv class DatabaseView(View): @@ -57,7 +56,7 @@ class DatabaseView(View): sql = (request.args.get("sql") or "").strip() if sql: - return await query_view(request, datasette) + return await QueryView()(request, datasette) if format_ not in ("html", "json"): raise NotFound("Invalid format: {}".format(format_)) @@ -65,10 +64,6 @@ class DatabaseView(View): metadata = (datasette.metadata("databases") or {}).get(database, {}) datasette.update_with_inherited_metadata(metadata) - table_counts = await db.table_counts(5) - hidden_table_names = set(await db.hidden_table_names()) - all_foreign_keys = await db.get_all_foreign_keys() - sql_views = [] for view_name in await db.view_names(): view_visible, view_private = await datasette.check_visibility( @@ -196,8 +191,13 @@ class QueryContext: # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_write: bool = field( - metadata={"help": "Boolean indicating if this canned query allows writes"} + canned_query_write: bool = field( + metadata={ + "help": "Boolean indicating if this is a canned query that allows writes" + } + ) + metadata: dict = field( + metadata={"help": "Metadata about the database or the canned query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -232,7 +232,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - metadata: dict = field(metadata={"help": "Metadata about the query/database"}) database_color: Callable = field( metadata={"help": "Function that returns a color for a given database name"} ) @@ -242,6 +241,12 @@ class QueryContext: alternate_url_json: str = field( metadata={"help": "URL for alternate JSON version of this page"} ) + # TODO: refactor this to somewhere else, probably ds.render_template() + select_templates: list = field( + metadata={ + "help": "List of templates that were considered for rendering this page" + } + ) async def get_tables(datasette, request, db): @@ -320,287 +325,105 @@ async def database_download(request, datasette): ) -async def query_view( - request, - datasette, - # canned_query=None, - # _size=None, - # named_parameters=None, - # write=False, -): - db = await datasette.resolve_database(request) - database = db.name - # Flattened because of ?sql=&name1=value1&name2=value2 feature - params = {key: request.args.get(key) for key in request.args} - sql = None - if "sql" in params: - sql = params.pop("sql") - if "_shape" in params: - params.pop("_shape") +class QueryView(View): + async def post(self, request, datasette): + from datasette.app import TableNotFound - # extras come from original request.args to avoid being flattened - extras = request.args.getlist("_extra") + db = await datasette.resolve_database(request) - # TODO: Behave differently for canned query here: - await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - - _, private = await datasette.check_visibility( - request.actor, - permissions=[ - ("view-database", database), - "view-instance", - ], - ) - - extra_args = {} - if params.get("_timelimit"): - extra_args["custom_time_limit"] = int(params["_timelimit"]) - - format_ = request.url_vars.get("format") or "html" - query_error = None - try: - validate_sql_select(sql) - results = await datasette.execute( - database, sql, params, truncate=True, **extra_args - ) - columns = results.columns - rows = results.rows - except QueryInterrupted as ex: - raise DatasetteError( - textwrap.dedent( - """ -

    SQL query took too long. The time limit is controlled by the - sql_time_limit_ms - configuration option.

    - - - """.format( - markupsafe.escape(ex.sql) - ) - ).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except sqlite3.DatabaseError as ex: - query_error = str(ex) - results = None - rows = [] - columns = [] - except (sqlite3.OperationalError, InvalidSql) as ex: - raise DatasetteError(str(ex), title="Invalid SQL", status=400) - except sqlite3.OperationalError as ex: - raise DatasetteError(str(ex)) - except DatasetteError: - raise - - # Handle formats from plugins - if format_ == "csv": - - async def fetch_data_for_csv(request, _next=None): - results = await db.execute(sql, params, truncate=True) - data = {"rows": results.rows, "columns": results.columns} - return data, None, None - - return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - datasette.renderers[format_][0], - datasette=datasette, - columns=columns, - rows=rows, - sql=sql, - query_name=None, - database=database, - table=None, - request=request, - view_name="table", - truncated=results.truncated if results else False, - error=query_error, - # These will be deprecated in Datasette 1.0: - args=request.args, - data={"rows": rows, "columns": columns}, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code") or 200, - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), + # We must be a canned query + table_found = False + try: + await datasette.resolve_table(request) + table_found = True + except TableNotFound as table_not_found: + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor ) - elif isinstance(result, Response): - r = result - # if status_code is not None: - # # Over-ride the status code - # r.status = status_code - else: - assert False, f"{result} should be dict or Response" - elif format_ == "html": - headers = {} - templates = [f"query-{to_css_class(database)}.html", "query.html"] - template = datasette.jinja_env.select_template(templates) - alternate_url_json = datasette.absolute_url( - request, - datasette.urls.path(path_with_format(request=request, format="json")), - ) - data = {} - headers.update( - { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - } - ) - metadata = (datasette.metadata("databases") or {}).get(database, {}) - datasette.update_with_inherited_metadata(metadata) + if canned_query is None: + raise + if table_found: + # That should not have happened + raise DatasetteError("Unexpected table found on POST", status=404) - renderers = {} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name="database", + # If database is immutable, return an error + if not db.is_mutable: + raise Forbidden("Database is immutable") + + # Process the POST + body = await request.post_body() + body = body.decode("utf-8").strip() + if body.startswith("{") and body.endswith("}"): + params = json.loads(body) + # But we want key=value strings + for key, value in params.items(): + params[key] = str(value) + else: + params = dict(parse_qsl(body, keep_blank_values=True)) + # Should we return JSON? + should_return_json = ( + request.headers.get("accept") == "application/json" + or request.args.get("_json") + or params.get("_json") + ) + params_for_query = MagicParameters(params, request, datasette) + ok = None + redirect_url = None + try: + cursor = await db.execute_write(canned_query["sql"], params_for_query) + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + message_type = datasette.INFO + redirect_url = canned_query.get("on_success_redirect") + ok = True + except Exception as ex: + message = canned_query.get("on_error_message") or str(ex) + message_type = datasette.ERROR + redirect_url = canned_query.get("on_error_redirect") + ok = False + if should_return_json: + return Response.json( + { + "ok": ok, + "message": message, + "redirect": redirect_url, + } ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format(request=request, format=key) - ) - - allow_execute_sql = await datasette.permission_allowed( - request.actor, "execute-sql", database - ) - - show_hide_hidden = "" - if metadata.get("hide_sql"): - if bool(params.get("_show_sql")): - show_hide_link = path_with_removed_args(request, {"_show_sql"}) - show_hide_text = "hide" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_show_sql": 1}) - show_hide_text = "show" else: - if bool(params.get("_hide_sql")): - show_hide_link = path_with_removed_args(request, {"_hide_sql"}) - show_hide_text = "show" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) - show_hide_text = "hide" - hide_sql = show_hide_text == "show" + datasette.add_message(request, message, message_type) + return Response.redirect(redirect_url or request.path) - # Extract any :named parameters - named_parameters = await derive_named_parameters( - datasette.get_database(database), sql - ) - named_parameter_values = { - named_parameter: params.get(named_parameter) or "" - for named_parameter in named_parameters - if not named_parameter.startswith("_") - } + async def get(self, request, datasette): + from datasette.app import TableNotFound - # Set to blank string if missing from params - for named_parameter in named_parameters: - if named_parameter not in params and not named_parameter.startswith("_"): - params[named_parameter] = "" - - r = Response.html( - await datasette.render_template( - template, - QueryContext( - database=database, - query={ - "sql": sql, - "params": params, - }, - canned_query=None, - private=private, - canned_write=False, - db_is_immutable=not db.is_mutable, - error=query_error, - hide_sql=hide_sql, - show_hide_link=datasette.urls.path(show_hide_link), - show_hide_text=show_hide_text, - editable=True, # TODO - allow_execute_sql=allow_execute_sql, - tables=await get_tables(datasette, request, db), - named_parameter_values=named_parameter_values, - edit_sql_url="todo", - display_rows=await display_rows( - datasette, database, request, rows, columns - ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, - columns=columns, - renderers=renderers, - url_csv=datasette.urls.path( - path_with_format( - request=request, format="csv", extra_qs={"_size": "max"} - ) - ), - show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=metadata, - database_color=lambda _: "#ff0000", - alternate_url_json=alternate_url_json, - ), - request=request, - view_name="database", - ), - headers=headers, - ) - else: - assert False, "Invalid format: {}".format(format_) - if datasette.cors: - add_cors_headers(r.headers) - return r - - -class QueryView(DataView): - async def data( - self, - request, - sql, - editable=True, - canned_query=None, - metadata=None, - _size=None, - named_parameters=None, - write=False, - default_labels=None, - ): - db = await self.ds.resolve_database(request) + db = await datasette.resolve_database(request) database = db.name - params = {key: request.args.get(key) for key in request.args} - if "sql" in params: - params.pop("sql") - if "_shape" in params: - params.pop("_shape") + + # Are we a canned query? + canned_query = None + canned_query_write = False + if "table" in request.url_vars: + try: + await datasette.resolve_table(request) + except TableNotFound as table_not_found: + # Was this actually a canned query? + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor + ) + if canned_query is None: + raise + canned_query_write = bool(canned_query.get("write")) private = False if canned_query: # Respect canned query permissions - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, permissions=[ - ("view-query", (database, canned_query)), + ("view-query", (database, canned_query["name"])), ("view-database", database), "view-instance", ], @@ -609,18 +432,32 @@ class QueryView(DataView): raise Forbidden("You do not have permission to view this query") else: - await self.ds.ensure_permissions(request.actor, [("execute-sql", database)]) + await datasette.ensure_permissions( + request.actor, [("execute-sql", database)] + ) + + # Flattened because of ?sql=&name1=value1&name2=value2 feature + params = {key: request.args.get(key) for key in request.args} + sql = None + + if canned_query: + sql = canned_query["sql"] + elif "sql" in params: + sql = params.pop("sql") # Extract any :named parameters - named_parameters = named_parameters or await derive_named_parameters( - self.ds.get_database(database), sql - ) + named_parameters = [] + if canned_query and canned_query.get("params"): + named_parameters = canned_query["params"] + if not named_parameters: + named_parameters = await derive_named_parameters( + datasette.get_database(database), sql + ) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters if not named_parameter.startswith("_") } - # Set to blank string if missing from params for named_parameter in named_parameters: if named_parameter not in params and not named_parameter.startswith("_"): @@ -629,212 +466,159 @@ class QueryView(DataView): extra_args = {} if params.get("_timelimit"): extra_args["custom_time_limit"] = int(params["_timelimit"]) - if _size: - extra_args["page_size"] = _size - templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: - templates.insert( - 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", - ) + format_ = request.url_vars.get("format") or "html" query_error = None + results = None + rows = [] + columns = [] - # Execute query - as write or as read - if write: - if request.method == "POST": - # If database is immutable, return an error - if not db.is_mutable: - raise Forbidden("Database is immutable") - body = await request.post_body() - body = body.decode("utf-8").strip() - if body.startswith("{") and body.endswith("}"): - params = json.loads(body) - # But we want key=value strings - for key, value in params.items(): - params[key] = str(value) - else: - params = dict(parse_qsl(body, keep_blank_values=True)) - # Should we return JSON? - should_return_json = ( - request.headers.get("accept") == "application/json" - or request.args.get("_json") - or params.get("_json") - ) - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params - ok = None - try: - cursor = await self.ds.databases[database].execute_write( - sql, params_for_query - ) - message = metadata.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) - message_type = self.ds.INFO - redirect_url = metadata.get("on_success_redirect") - ok = True - except Exception as e: - message = metadata.get("on_error_message") or str(e) - message_type = self.ds.ERROR - redirect_url = metadata.get("on_error_redirect") - ok = False - if should_return_json: - return Response.json( - { - "ok": ok, - "message": message, - "redirect": redirect_url, - } - ) - else: - self.ds.add_message(request, message, message_type) - return self.redirect(request, redirect_url or request.path) - else: + params_for_query = params - async def extra_template(): - return { - "request": request, - "db_is_immutable": not db.is_mutable, - "path_with_added_args": path_with_added_args, - "path_with_removed_args": path_with_removed_args, - "named_parameter_values": named_parameter_values, - "canned_query": canned_query, - "success_message": request.args.get("_success") or "", - "canned_write": True, - } - - return ( - { - "database": database, - "rows": [], - "truncated": False, - "columns": [], - "query": {"sql": sql, "params": params}, - "private": private, - }, - extra_template, - templates, - ) - else: # Not a write - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params + if not canned_query_write: try: - results = await self.ds.execute( + if not canned_query: + # For regular queries we only allow SELECT, plus other rules + validate_sql_select(sql) + else: + # Canned queries can run magic parameters + params_for_query = MagicParameters(params, request, datasette) + results = await datasette.execute( database, sql, params_for_query, truncate=True, **extra_args ) - columns = [r[0] for r in results.description] - except sqlite3.DatabaseError as e: - query_error = e + columns = results.columns + rows = results.rows + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent( + """ +

    SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

    + + + """.format( + markupsafe.escape(ex.sql) + ) + ).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except sqlite3.DatabaseError as ex: + query_error = str(ex) results = None + rows = [] columns = [] + except (sqlite3.OperationalError, InvalidSql) as ex: + raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) + except DatasetteError: + raise - allow_execute_sql = await self.ds.permission_allowed( - request.actor, "execute-sql", database - ) + # Handle formats from plugins + if format_ == "csv": - async def extra_template(): - display_rows = [] - truncate_cells = self.ds.setting("truncate_cells_html") - for row in results.rows if results else []: - display_row = [] - for column, value in zip(results.columns, row): - display_value = value - # Let the plugins have a go - # pylint: disable=no-member - plugin_display_value = None - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=None, - database=database, - datasette=self.ds, - request=request, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value is not None: - display_value = plugin_display_value - else: - if value in ("", None): - display_value = markupsafe.Markup(" ") - elif is_url(str(display_value).strip()): - display_value = markupsafe.Markup( - '{truncated_url}'.format( - url=markupsafe.escape(value.strip()), - truncated_url=markupsafe.escape( - truncate_url(value.strip(), truncate_cells) - ), - ) - ) - elif isinstance(display_value, bytes): - blob_url = path_with_format( - request=request, - format="blob", - extra_qs={ - "_blob_column": column, - "_blob_hash": hashlib.sha256( - display_value - ).hexdigest(), - }, - ) - formatted = format_bytes(len(value)) - display_value = markupsafe.Markup( - '<Binary: {:,} byte{}>'.format( - blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", - len(value), - "" if len(value) == 1 else "s", - ) - ) - else: - display_value = str(value) - if truncate_cells and len(display_value) > truncate_cells: - display_value = ( - display_value[:truncate_cells] + "\u2026" - ) - display_row.append(display_value) - display_rows.append(display_row) + async def fetch_data_for_csv(request, _next=None): + results = await db.execute(sql, params, truncate=True) + data = {"rows": results.rows, "columns": results.columns} + return data, None, None - # Show 'Edit SQL' button only if: - # - User is allowed to execute SQL - # - SQL is an approved SELECT statement - # - No magic parameters, so no :_ in the SQL string - edit_sql_url = None - is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - self.ds.urls.database(database) - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } - ) + return await stream_csv(datasette, fetch_data_for_csv, request, db.name) + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=canned_query["name"] if canned_query else None, + database=database, + table=None, + request=request, + view_name="table", + truncated=results.truncated if results else False, + error=query_error, + # These will be deprecated in Datasette 1.0: + args=request.args, + data={"rows": rows, "columns": columns}, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + data = {} + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + renderers = {} + for key, (_, can_render) in datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=datasette, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name="database", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = datasette.urls.path( + path_with_format(request=request, format=key) + ) + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + show_hide_hidden = "" - if metadata.get("hide_sql"): + if canned_query and canned_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -855,42 +639,86 @@ class QueryView(DataView): show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) show_hide_text = "hide" hide_sql = show_hide_text == "show" - return { - "display_rows": display_rows, - "custom_sql": True, - "named_parameter_values": named_parameter_values, - "editable": editable, - "canned_query": canned_query, - "edit_sql_url": edit_sql_url, - "metadata": metadata, - "settings": self.ds.settings_dict(), - "request": request, - "show_hide_link": self.ds.urls.path(show_hide_link), - "show_hide_text": show_hide_text, - "show_hide_hidden": markupsafe.Markup(show_hide_hidden), - "hide_sql": hide_sql, - "table_columns": await _table_columns(self.ds, database) - if allow_execute_sql - else {}, - } - return ( - { - "ok": not query_error, - "database": database, - "query_name": canned_query, - "rows": results.rows if results else [], - "truncated": results.truncated if results else False, - "columns": columns, - "query": {"sql": sql, "params": params}, - "error": str(query_error) if query_error else None, - "private": private, - "allow_execute_sql": allow_execute_sql, - }, - extra_template, - templates, - 400 if query_error else 200, - ) + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + r = Response.html( + await datasette.render_template( + template, + QueryContext( + database=database, + query={ + "sql": sql, + "params": params, + }, + canned_query=canned_query["name"] if canned_query else None, + private=private, + canned_query_write=canned_query_write, + db_is_immutable=not db.is_mutable, + error=query_error, + hide_sql=hide_sql, + show_hide_link=datasette.urls.path(show_hide_link), + show_hide_text=show_hide_text, + editable=not canned_query, + allow_execute_sql=allow_execute_sql, + tables=await get_tables(datasette, request, db), + named_parameter_values=named_parameter_values, + edit_sql_url=edit_sql_url, + display_rows=await display_rows( + datasette, database, request, rows, columns + ), + table_columns=await _table_columns(datasette, database) + if allow_execute_sql + else {}, + columns=columns, + renderers=renderers, + url_csv=datasette.urls.path( + path_with_format( + request=request, format="csv", extra_qs={"_size": "max"} + ) + ), + show_hide_hidden=markupsafe.Markup(show_hide_hidden), + metadata=canned_query or metadata, + database_color=lambda _: "#ff0000", + alternate_url_json=alternate_url_json, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="database", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + if datasette.cors: + add_cors_headers(r.headers) + return r class MagicParameters(dict): diff --git a/datasette/views/table.py b/datasette/views/table.py index 77acfd95..28264e92 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -21,7 +20,6 @@ from datasette.utils import ( tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, path_from_row_pks, path_with_added_args, @@ -38,7 +36,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils -from .base import BaseView, DataView, DatasetteError, ureg, _error, stream_csv +from .base import BaseView, DatasetteError, ureg, _error, stream_csv from .database import QueryView LINK_WITH_LABEL = ( @@ -698,57 +696,6 @@ async def table_view(datasette, request): return response -class CannedQueryView(DataView): - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as e: - # Was this actually a canned query? - canned_query = await self.ds.get_canned_query( - e.database_name, e.table, request.actor - ) - if canned_query: - # Handle POST to a canned query - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=e.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - - return Response.text("Method not allowed", status=405) - - async def data(self, request, **kwargs): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as not_found: - canned_query = await self.ds.get_canned_query( - not_found.database_name, not_found.table, request.actor - ) - if canned_query: - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=not_found.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - else: - raise - - async def table_view_traced(datasette, request): from datasette.app import TableNotFound @@ -761,10 +708,7 @@ async def table_view_traced(datasette, request): ) # If this is a canned query, not a table, then dispatch to QueryView instead if canned_query: - if request.method == "POST": - return await CannedQueryView(datasette).post(request) - else: - return await CannedQueryView(datasette).get(request) + return await QueryView()(request, datasette) else: raise diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index d6a88733..e9ad3239 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -95,12 +95,12 @@ def test_insert(canned_write_client): csrftoken_from=True, cookies={"foo": "bar"}, ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] messages = canned_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) - assert [["Query executed, 1 row affected", 1]] == messages + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" @pytest.mark.parametrize( @@ -382,11 +382,11 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c def test_canned_write_custom_template(canned_write_client): response = canned_write_client.get("/data/update_name") assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text assert ( "" in response.text ) - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text # And test for link rel=alternate while we're here: assert ( '' From 8920d425f4d417cfd998b61016c5ff3530cd34e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 10:20:58 -0700 Subject: [PATCH 1101/1866] 1.0a3 release notes, smaller changes section - refs #2135 --- docs/changelog.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee48d075..b4416f94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,25 @@ Changelog ========= +.. _v1_0_a3: + +1.0a3 (2023-08-09) +------------------ + +This alpha release previews the updated design for Datasette's default JSON API. + +Smaller changes +~~~~~~~~~~~~~~~ + +- Datasette documentation now shows YAML examples for :ref:`metadata` by default, with a tab interface for switching to JSON. (:issue:`1153`) +- :ref:`plugin_register_output_renderer` plugins now have access to ``error`` and ``truncated`` arguments, allowing them to display error messages and take into account truncated results. (:issue:`2130`) +- ``render_cell()`` plugin hook now also supports an optional ``request`` argument. (:issue:`2007`) +- New ``Justfile`` to support development workflows for Datasette using `Just `__. +- ``datasette.render_template()`` can now accepts a ``datasette.views.Context`` subclass as an alternative to a dictionary. (:issue:`2127`) +- ``datasette install -e path`` option for editable installations, useful while developing plugins. (:issue:`2106`) +- When started with the ``--cors`` option Datasette now serves an ``Access-Control-Max-Age: 3600`` header, ensuring CORS OPTIONS requests are repeated no more than once an hour. (:issue:`2079`) +- Fixed a bug where the ``_internal`` database could display ``None`` instead of ``null`` for in-memory databases. (:issue:`1970`) + .. _v0_64_2: 0.64.2 (2023-03-08) From e34d09c6ec16ff5e7717e112afdad67f7c05a62a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:01:59 -0700 Subject: [PATCH 1102/1866] Don't include columns in query JSON, refs #2136 --- datasette/renderer.py | 8 +++++++- datasette/views/database.py | 2 +- tests/test_api.py | 1 - tests/test_cli_serve_get.py | 11 ++++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0bd74e81..224031a7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -27,7 +27,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): return new_rows -def json_renderer(args, data, error, truncated=None): +def json_renderer(request, args, data, error, truncated=None): """Render a response as JSON""" status_code = 200 @@ -106,6 +106,12 @@ def json_renderer(args, data, error, truncated=None): "status": 400, "title": None, } + + # Don't include "columns" in output + # https://github.com/simonw/datasette/issues/2136 + if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + data.pop("columns", None) + # Handle _nl option for _shape=array nl = args.get("_nl", "") if nl and shape == "array": diff --git a/datasette/views/database.py b/datasette/views/database.py index 658c35e6..cf76f3c2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -548,7 +548,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"rows": rows, "columns": columns}, + data={"ok": True, "rows": rows, "columns": columns}, ) if asyncio.iscoroutine(result): result = await result diff --git a/tests/test_api.py b/tests/test_api.py index 28415a0b..f96f571e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -649,7 +649,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ], - "columns": ["content"], "ok": True, "truncated": False, } diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 2e0390bb..dc7fc1e2 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -34,11 +34,12 @@ def test_serve_with_get(tmp_path_factory): "/_memory.json?sql=select+sqlite_version()", ], ) - assert 0 == result.exit_code, result.output - assert { - "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + assert result.exit_code == 0, result.output + data = json.loads(result.output) + # Should have a single row with a single column + assert len(data["rows"]) == 1 + assert list(data["rows"][0].keys()) == ["sqlite_version()"] + assert set(data.keys()) == {"rows", "ok", "truncated"} # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From 856ca68d94708c6e94673cb6bc28bf3e3ca17845 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:04:40 -0700 Subject: [PATCH 1103/1866] Update default JSON representation docs, refs #2135 --- docs/json_api.rst | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index c273c2a8..16b997eb 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,10 +9,10 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. -.. _json_api_shapes: +.. _json_api_default: -Different shapes ----------------- +Default representation +---------------------- The default JSON representation of data from a SQLite table or custom query looks like this: @@ -21,7 +21,6 @@ looks like this: { "ok": true, - "next": null, "rows": [ { "id": 3, @@ -39,13 +38,22 @@ looks like this: "id": 1, "name": "San Francisco" } - ] + ], + "truncated": false } -The ``rows`` key is a list of objects, each one representing a row. ``next`` indicates if -there is another page, and ``ok`` is always ``true`` if an error did not occur. +``"ok"`` is always ``true`` if an error did not occur. -If ``next`` is present then the next page in the pagination set can be retrieved using ``?_next=VALUE``. +The ``"rows"`` key is a list of objects, each one representing a row. + +The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). + +For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. + +.. _json_api_shapes: + +Different shapes +---------------- The ``_shape`` parameter can be used to access alternative formats for the ``rows`` key which may be more convenient for your application. There are three From 90cb9ca58d910f49e8f117bbdd94df6f0855cf99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:11:16 -0700 Subject: [PATCH 1104/1866] JSON changes in release notes, refs #2135 --- docs/changelog.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4416f94..4c70855b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,40 @@ Changelog 1.0a3 (2023-08-09) ------------------ -This alpha release previews the updated design for Datasette's default JSON API. +This alpha release previews the updated design for Datasette's default JSON API. (:issue:`782`) + +The new :ref:`default JSON representation ` for both table pages (``/dbname/table.json``) and arbitrary SQL queries (``/dbname.json?sql=...``) is now shaped like this: + +.. code-block:: json + + { + "ok": true, + "rows": [ + { + "id": 3, + "name": "Detroit" + }, + { + "id": 2, + "name": "Los Angeles" + }, + { + "id": 4, + "name": "Memnonia" + }, + { + "id": 1, + "name": "San Francisco" + } + ], + "truncated": false + } + +Tables will include an additional ``"next"`` key for pagination, which can be passed to ``?_next=`` to fetch the next page of results. + +The various ``?_shape=`` options continue to work as before - see :ref:`json_api_shapes` for details. + +A new ``?_extra=`` mechanism is available for tables, but has not yet been stabilized or documented. Details on that are available in :issue:`262`. Smaller changes ~~~~~~~~~~~~~~~ From 19ab4552e212c9845a59461cc73e82d5ae8c278a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:13:11 -0700 Subject: [PATCH 1105/1866] Release 1.0a3 Closes #2135 Refs #262, #782, #1153, #1970, #2007, #2079, #2106, #2127, #2130 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 3b81ab21..61dee464 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a2" +__version__ = "1.0a3" __version_info__ = tuple(__version__.split(".")) From 4a42476bb7ce4c5ed941f944115dedd9bce34656 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 15:04:16 -0700 Subject: [PATCH 1106/1866] datasette plugins --requirements, closes #2133 --- datasette/cli.py | 12 ++++++++++-- docs/cli-reference.rst | 1 + docs/plugins.rst | 32 ++++++++++++++++++++++++++++---- tests/test_cli.py | 3 +++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 32266888..21fd25d6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -223,15 +223,23 @@ pm.hook.publish_subcommand(publish=publish) @cli.command() @click.option("--all", help="Include built-in default plugins", is_flag=True) +@click.option( + "--requirements", help="Output requirements.txt of installed plugins", is_flag=True +) @click.option( "--plugins-dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Path to directory containing custom plugins", ) -def plugins(all, plugins_dir): +def plugins(all, requirements, plugins_dir): """List currently installed plugins""" app = Datasette([], plugins_dir=plugins_dir) - click.echo(json.dumps(app._plugins(all=all), indent=4)) + if requirements: + for plugin in app._plugins(): + if plugin["version"]: + click.echo("{}=={}".format(plugin["name"], plugin["version"])) + else: + click.echo(json.dumps(app._plugins(all=all), indent=4)) @cli.command() diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2177fc9e..7a96d311 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -282,6 +282,7 @@ Output JSON showing all currently installed plugins, their versions, whether the Options: --all Include built-in default plugins + --requirements Output requirements.txt of installed plugins --plugins-dir DIRECTORY Path to directory containing custom plugins --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 979f94dd..19bfdd0c 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -90,7 +90,12 @@ You can see a list of installed plugins by navigating to the ``/-/plugins`` page You can also use the ``datasette plugins`` command:: - $ datasette plugins + datasette plugins + +Which outputs: + +.. code-block:: json + [ { "name": "datasette_json_html", @@ -107,7 +112,8 @@ You can also use the ``datasette plugins`` command:: cog.out("\n") result = CliRunner().invoke(cli.cli, ["plugins", "--all"]) # cog.out() with text containing newlines was unindenting for some reason - cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::\n") + cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:\n") + cog.outl(".. code-block:: json\n") plugins = [p for p in json.loads(result.output) if p["name"].startswith("datasette.")] indented = textwrap.indent(json.dumps(plugins, indent=4), " ") for line in indented.split("\n"): @@ -115,7 +121,9 @@ You can also use the ``datasette plugins`` command:: cog.out("\n\n") .. ]]] -If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: +If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette: + +.. code-block:: json [ { @@ -236,6 +244,22 @@ If you run ``datasette plugins --all`` it will include default plugins that ship You can add the ``--plugins-dir=`` option to include any plugins found in that directory. +Add ``--requirements`` to output a list of installed plugins that can then be installed in another Datasette instance using ``datasette install -r requirements.txt``:: + + datasette plugins --requirements + +The output will look something like this:: + + datasette-codespaces==0.1.1 + datasette-graphql==2.2 + datasette-json-html==1.0.1 + datasette-pretty-json==0.2.2 + datasette-x-forwarded-host==0.1 + +To write that to a ``requirements.txt`` file, run this:: + + datasette plugins --requirements > requirements.txt + .. _plugins_configuration: Plugin configuration @@ -390,7 +414,7 @@ Any values embedded in ``metadata.yaml`` will be visible to anyone who views the If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ diff --git a/tests/test_cli.py b/tests/test_cli.py index 75724f61..056e2821 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,6 +108,9 @@ def test_plugins_cli(app_client): assert set(names).issuperset({p["name"] for p in EXPECTED_PLUGINS}) # And the following too: assert set(names).issuperset(DEFAULT_PLUGINS) + # --requirements should be empty because there are no installed non-plugins-dir plugins + result3 = runner.invoke(cli, ["plugins", "--requirements"]) + assert result3.output == "" def test_metadata_yaml(): From a3593c901580ea50854c3e0774b0ba0126e8a76f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:32:07 -0700 Subject: [PATCH 1107/1866] on_success_message_sql, closes #2138 --- datasette/views/database.py | 29 ++++++++++++++++---- docs/sql_queries.rst | 21 ++++++++++---- tests/test_canned_queries.py | 53 +++++++++++++++++++++++++++++++----- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cf76f3c2..79b3f88d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -360,6 +360,10 @@ class QueryView(View): params[key] = str(value) else: params = dict(parse_qsl(body, keep_blank_values=True)) + + # Don't ever send csrftoken as a SQL parameter + params.pop("csrftoken", None) + # Should we return JSON? should_return_json = ( request.headers.get("accept") == "application/json" @@ -371,12 +375,27 @@ class QueryView(View): redirect_url = None try: cursor = await db.execute_write(canned_query["sql"], params_for_query) - message = canned_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) + # success message can come from on_success_message or on_success_message_sql + message = None message_type = datasette.INFO + on_success_message_sql = canned_query.get("on_success_message_sql") + if on_success_message_sql: + try: + message_result = ( + await db.execute(on_success_message_sql, params_for_query) + ).first() + if message_result: + message = message_result[0] + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) + message_type = datasette.ERROR + if not message: + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + redirect_url = canned_query.get("on_success_redirect") ok = True except Exception as ex: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 3c2cb228..1ae07e1f 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -392,6 +392,7 @@ This configuration will create a page at ``/mydatabase/add_name`` displaying a f You can customize how Datasette represents success and errors using the following optional properties: - ``on_success_message`` - the message shown when a query is successful +- ``on_success_message_sql`` - alternative to ``on_success_message``: a SQL query that should be executed to generate the message - ``on_success_redirect`` - the path or URL the user is redirected to on success - ``on_error_message`` - the message shown when a query throws an error - ``on_error_redirect`` - the path or URL the user is redirected to on error @@ -405,11 +406,12 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": ["name"], "write": True, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", - "on_error_redirect": "/mydatabase" + "on_error_redirect": "/mydatabase", } } } @@ -426,8 +428,10 @@ For example: queries: add_name: sql: INSERT INTO names (name) VALUES (:name) + params: + - name write: true - on_success_message: Name inserted + on_success_message_sql: 'select ''Name inserted: '' || :name' on_success_redirect: /mydatabase/names on_error_message: Name insert failed on_error_redirect: /mydatabase @@ -443,8 +447,11 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": [ + "name" + ], "write": true, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", "on_error_redirect": "/mydatabase" @@ -455,10 +462,12 @@ For example: } .. [[[end]]] -You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. +You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. ``"params"`` is not necessary in the above example, since without it ``"name"`` would be automatically detected from the query. You can pre-populate form fields when the page first loads using a query string, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query. +If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. + .. _canned_queries_magic_parameters: Magic parameters @@ -589,7 +598,7 @@ The JSON response will look like this: "redirect": "/data/add_name" } -The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index e9ad3239..5256c24c 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -31,9 +31,15 @@ def canned_write_client(tmpdir): }, "add_name_specify_id": { "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", "write": True, "on_error_redirect": "/data/add_name_specify_id?error", }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, "delete_name": { "sql": "delete from names where rowid = :rowid", "write": True, @@ -179,6 +185,34 @@ def test_insert_error(canned_write_client): ) +def test_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") assert '' in response.text @@ -232,21 +266,22 @@ def test_canned_query_permissions_on_database_page(canned_write_client): query_names = { q["name"] for q in canned_write_client.get("/data.json").json["queries"] } - assert { + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "from_hook", + "update_name", + "add_name_specify_id", + "from_async_hook", "canned_read", "add_name", - "add_name_specify_id", - "update_name", - "from_async_hook", - "from_hook", - } == query_names + } # With auth shows four response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, ) - assert 200 == response.status + assert response.status == 200 query_names_and_private = sorted( [ {"name": q["name"], "private": q["private"]} @@ -257,6 +292,10 @@ def test_canned_query_permissions_on_database_page(canned_write_client): assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, {"name": "canned_read", "private": False}, {"name": "delete_name", "private": True}, {"name": "from_async_hook", "private": False}, From 33251d04e78d575cca62bb59069bb43a7d924746 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:56:27 -0700 Subject: [PATCH 1108/1866] Canned query write counters demo, refs #2134 --- .github/workflows/deploy-latest.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index ed60376c..4746aa07 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,6 +57,36 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db + - name: And the counters writable canned query demo + run: | + cat > plugins/counters.py < Date: Thu, 10 Aug 2023 22:16:19 -0700 Subject: [PATCH 1109/1866] Fixed display of database color Closes #2139, closes #2119 --- datasette/database.py | 7 +++++++ datasette/templates/database.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/row.html | 2 +- datasette/templates/table.html | 2 +- datasette/views/base.py | 4 ---- datasette/views/database.py | 8 +++----- datasette/views/index.py | 4 +--- datasette/views/row.py | 4 +++- datasette/views/table.py | 2 +- tests/test_html.py | 20 ++++++++++++++++++++ 11 files changed, 39 insertions(+), 18 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d8043c24..af39ac9e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,6 +1,7 @@ import asyncio from collections import namedtuple from pathlib import Path +import hashlib import janus import queue import sys @@ -62,6 +63,12 @@ class Database: } return self._cached_table_counts + @property + def color(self): + if self.hash: + return self.hash[:6] + return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + def suggest_name(self): if self.path: return Path(self.path).stem diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 7acf0369..3d4dae07 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -10,7 +10,7 @@ {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} {% block content %} -
    +

    {{ metadata.title or database }}{% if private %} 🔒{% endif %}

    {% set links = database_actions() %}{% if links %}
    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index fc3b8527..b8f06f84 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -28,7 +28,7 @@

    This query cannot be executed because the database is immutable.

    {% endif %} -

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 1d1b0bfd..4d179a85 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -20,7 +20,7 @@ {% endblock %} {% block content %} -

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    +

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 0be1998c..88580e52 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -21,7 +21,7 @@ {% endblock %} {% block content %} -
    +

    {{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

    {% set links = table_actions() %}{% if links %}
    diff --git a/datasette/views/base.py b/datasette/views/base.py index da5c55ad..0080b33c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -102,9 +102,6 @@ class BaseView: response.body = b"" return response - def database_color(self, database): - return "ff0000" - async def method_not_allowed(self, request): if ( request.path.endswith(".json") @@ -150,7 +147,6 @@ class BaseView: template_context = { **context, **{ - "database_color": self.database_color, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates diff --git a/datasette/views/database.py b/datasette/views/database.py index 79b3f88d..d9abc38a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -146,6 +146,7 @@ class DatabaseView(View): template = datasette.jinja_env.select_template(templates) context = { **json_data, + "database_color": db.color, "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, @@ -154,7 +155,6 @@ class DatabaseView(View): and not db.is_mutable and not db.is_memory, "attached_databases": attached_databases, - "database_color": lambda _: "#ff0000", "alternate_url_json": alternate_url_json, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" @@ -179,6 +179,7 @@ class DatabaseView(View): @dataclass class QueryContext: database: str = field(metadata={"help": "The name of the database being queried"}) + database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) @@ -232,9 +233,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - database_color: Callable = field( - metadata={"help": "Function that returns a color for a given database name"} - ) table_columns: dict = field( metadata={"help": "Dictionary of table name to list of column names"} ) @@ -689,6 +687,7 @@ class QueryView(View): template, QueryContext( database=database, + database_color=db.color, query={ "sql": sql, "params": params, @@ -721,7 +720,6 @@ class QueryView(View): ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), metadata=canned_query or metadata, - database_color=lambda _: "#ff0000", alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/datasette/views/index.py b/datasette/views/index.py index df411c4a..95b29302 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -105,9 +105,7 @@ class IndexView(BaseView): { "name": name, "hash": db.hash, - "color": db.hash[:6] - if db.hash - else hashlib.md5(name.encode("utf8")).hexdigest()[:6], + "color": db.color, "path": self.ds.urls.database(name), "tables_and_views_truncated": tables_and_views_truncated, "tables_and_views_more": (len(visible_tables) + len(views)) diff --git a/datasette/views/row.py b/datasette/views/row.py index 6e09f30e..8f07a662 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -18,7 +18,8 @@ class RowView(DataView): async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) - database = resolved.db.name + db = resolved.db + database = db.name table = resolved.table pk_values = resolved.pk_values @@ -60,6 +61,7 @@ class RowView(DataView): "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), + "database_color": db.color, "display_columns": display_columns, "display_rows": display_rows, "custom_table_templates": [ diff --git a/datasette/views/table.py b/datasette/views/table.py index 28264e92..6df8b915 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1408,7 +1408,7 @@ async def table_view_data( return table_name async def extra_database_color(): - return lambda _: "ff0000" + return db.color async def extra_form_hidden_args(): form_hidden_args = [] diff --git a/tests/test_html.py b/tests/test_html.py index 7856bc27..ffc2aef1 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1103,3 +1103,23 @@ async def test_breadcrumbs_respect_permissions( assert actual == expected_links finally: ds_client.ds._metadata_local = orig + + +@pytest.mark.asyncio +async def test_database_color(ds_client): + expected_color = ds_client.ds.get_database("fixtures").color + # Should be something like #9403e5 + expected_fragments = ( + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), + ) + assert len(expected_color) == 6 + for path in ( + "/", + "/fixtures", + "/fixtures/facetable", + "/fixtures/paginated_view", + "/fixtures/pragma_cache_size", + ): + response = await ds_client.get(path) + assert any(fragment in response.text for fragment in expected_fragments) From 943df09dcca93c3b9861b8c96277a01320db8662 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Aug 2023 10:44:34 -0700 Subject: [PATCH 1110/1866] Remove all remaining "$ " prefixes from docs, closes #2140 Also document sqlite-utils create-view --- docs/authentication.rst | 5 ++++- docs/changelog.rst | 14 ++++++++++---- docs/cli-reference.rst | 5 ++++- docs/contributing.rst | 34 +++++++++++++++++++++++++--------- docs/custom_templates.rst | 8 ++++---- docs/deploying.rst | 2 +- docs/facets.rst | 7 +++++-- docs/full_text_search.rst | 4 ++-- docs/installation.rst | 38 ++++++++++++++++++++++++++++++++------ docs/plugin_hooks.rst | 2 +- docs/publish.rst | 4 ++-- docs/settings.rst | 12 ++++++------ docs/spatialite.rst | 5 ++++- docs/sql_queries.rst | 9 ++++++++- 14 files changed, 108 insertions(+), 41 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8864086f..814d2e67 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -32,7 +32,10 @@ The one exception is the "root" account, which you can sign into while using Dat To sign in as root, start Datasette using the ``--root`` command-line option, like this:: - $ datasette --root + datasette --root + +:: + http://127.0.0.1:8001/-/auth-token?token=786fc524e0199d70dc9a581d851f466244e114ca92f33aa3b42a139e9388daa7 INFO: Started server process [25801] INFO: Waiting for application startup. diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c70855b..c497ea9f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -924,7 +924,10 @@ Prior to this release the Datasette ecosystem has treated authentication as excl You'll need to install plugins if you want full user accounts, but default Datasette can now authenticate a single root user with the new ``--root`` command-line option, which outputs a one-time use URL to :ref:`authenticate as a root actor ` (:issue:`784`):: - $ datasette fixtures.db --root + datasette fixtures.db --root + +:: + http://127.0.0.1:8001/-/auth-token?token=5b632f8cd44b868df625f5a6e2185d88eea5b22237fd3cc8773f107cc4fd6477 INFO: Started server process [14973] INFO: Waiting for application startup. @@ -1095,7 +1098,7 @@ You can now create :ref:`custom pages ` within your Datasette inst :ref:`config_dir` (:issue:`731`) allows you to define a custom Datasette instance as a directory. So instead of running the following:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -1103,7 +1106,7 @@ You can now create :ref:`custom pages ` within your Datasette inst You can instead arrange your files in a single directory called ``my-project`` and run this:: - $ datasette my-project/ + datasette my-project/ Also in this release: @@ -1775,7 +1778,10 @@ In addition to the work on facets: Added new help section:: - $ datasette --help-config + datasette --help-config + + :: + Config options: default_page_size Default page size for the table view (default=100) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7a96d311..3dd991f3 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -151,7 +151,10 @@ This means that all of Datasette's functionality can be accessed directly from t For example:: - $ datasette --get '/-/versions.json' | jq . + datasette --get '/-/versions.json' | jq . + +.. code-block:: json + { "python": { "version": "3.8.5", diff --git a/docs/contributing.rst b/docs/contributing.rst index 697002a8..ef022a4d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -133,13 +133,19 @@ Running Black Black will be installed when you run ``pip install -e '.[test]'``. To test that your code complies with Black, run the following in your root ``datasette`` repository checkout:: - $ black . --check + black . --check + +:: + All done! ✨ 🍰 ✨ 95 files would be left unchanged. If any of your code does not conform to Black you can run this to automatically fix those problems:: - $ black . + black . + +:: + reformatted ../datasette/setup.py All done! ✨ 🍰 ✨ 1 file reformatted, 94 files left unchanged. @@ -160,11 +166,14 @@ Prettier To install Prettier, `install Node.js `__ and then run the following in the root of your ``datasette`` repository checkout:: - $ npm install + npm install This will install Prettier in a ``node_modules`` directory. You can then check that your code matches the coding style like so:: - $ npm run prettier -- --check + npm run prettier -- --check + +:: + > prettier > prettier 'datasette/static/*[!.min].js' "--check" @@ -174,7 +183,7 @@ This will install Prettier in a ``node_modules`` directory. You can then check t You can fix any problems by running:: - $ npm run fix + npm run fix .. _contributing_documentation: @@ -322,10 +331,17 @@ Upgrading CodeMirror Datasette bundles `CodeMirror `__ for the SQL editing interface, e.g. on `this page `__. Here are the steps for upgrading to a new version of CodeMirror: +* Install the packages with:: -* Install the packages with `npm i codemirror @codemirror/lang-sql` -* Build the bundle using the version number from package.json with: + npm i codemirror @codemirror/lang-sql - node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js -f iife -n cm -o datasette/static/cm-editor-6.0.1.bundle.js -p @rollup/plugin-node-resolve -p @rollup/plugin-terser +* Build the bundle using the version number from package.json with:: -* Update version reference in the `codemirror.html` template \ No newline at end of file + node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js \ + -f iife \ + -n cm \ + -o datasette/static/cm-editor-6.0.1.bundle.js \ + -p @rollup/plugin-node-resolve \ + -p @rollup/plugin-terser + +* Update the version reference in the ``codemirror.html`` template. \ No newline at end of file diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index f9d0b5f5..c0f64cb5 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -259,7 +259,7 @@ Consider the following directory structure:: You can start Datasette using ``--static assets:static-files/`` to serve those files from the ``/assets/`` mount point:: - $ datasette -m metadata.json --static assets:static-files/ --memory + datasette -m metadata.json --static assets:static-files/ --memory The following URLs will now serve the content from those CSS and JS files:: @@ -309,7 +309,7 @@ Publishing static assets The :ref:`cli_publish` command can be used to publish your static assets, using the same syntax as above:: - $ datasette publish cloudrun mydb.db --static assets:static-files/ + datasette publish cloudrun mydb.db --static assets:static-files/ This will upload the contents of the ``static-files/`` directory as part of the deployment, and configure Datasette to correctly serve the assets from ``/assets/``. @@ -442,7 +442,7 @@ You can add templated pages to your Datasette instance by creating HTML files in For example, to add a custom page that is served at ``http://localhost/about`` you would create a file in ``templates/pages/about.html``, then start Datasette like this:: - $ datasette mydb.db --template-dir=templates/ + datasette mydb.db --template-dir=templates/ You can nest directories within pages to create a nested structure. To create a ``http://localhost:8001/about/map`` page you would create ``templates/pages/about/map.html``. @@ -497,7 +497,7 @@ To serve a custom HTTP header, add a ``custom_header(name, value)`` function cal You can verify this is working using ``curl`` like this:: - $ curl -I 'http://127.0.0.1:8001/teapot' + curl -I 'http://127.0.0.1:8001/teapot' HTTP/1.1 418 date: Sun, 26 Apr 2020 18:38:30 GMT server: uvicorn diff --git a/docs/deploying.rst b/docs/deploying.rst index c8552758..3754267d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -56,7 +56,7 @@ Create a file at ``/etc/systemd/system/datasette.service`` with the following co Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Datasette cookies such as the CSRF token cookie. You can generate a suitable value like so:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' This configuration will run Datasette against all database files contained in the ``/home/ubuntu/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. diff --git a/docs/facets.rst b/docs/facets.rst index dba232bf..fabc2664 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -260,14 +260,17 @@ Speeding up facets with indexes The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: - $ sqlite3 mydatabase.db + sqlite3 mydatabase.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state"); Or using the `sqlite-utils `__ command-line utility:: - $ sqlite-utils create-index mydatabase.db Food_Trucks state + sqlite-utils create-index mydatabase.db Food_Trucks state .. _facet_by_json_array: diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index c956865b..43cf7eff 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -177,14 +177,14 @@ Configuring FTS using sqlite-utils Here's how to use ``sqlite-utils`` to enable full-text search for an ``items`` table across the ``name`` and ``description`` columns:: - $ sqlite-utils enable-fts mydatabase.db items name description + sqlite-utils enable-fts mydatabase.db items name description Configuring FTS using csvs-to-sqlite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your data starts out in CSV files, you can use Datasette's companion tool `csvs-to-sqlite `__ to convert that file into a SQLite database and enable full-text search on specific columns. For a file called ``items.csv`` where you want full-text search to operate against the ``name`` and ``description`` columns you would run the following:: - $ csvs-to-sqlite items.csv items.db -f name -f description + csvs-to-sqlite items.csv items.db -f name -f description Configuring FTS by hand ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/installation.rst b/docs/installation.rst index 52f87863..1cd2fddf 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -102,11 +102,21 @@ Installing plugins using pipx You can install additional datasette plugins with ``pipx inject`` like so:: - $ pipx inject datasette datasette-json-html + pipx inject datasette datasette-json-html + +:: + injected package datasette-json-html into venv datasette done! ✨ 🌟 ✨ - $ datasette plugins +Then to confirm the plugin was installed correctly: + +:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-json-html", @@ -121,12 +131,18 @@ Upgrading packages using pipx You can upgrade your pipx installation to the latest release of Datasette using ``pipx upgrade datasette``:: - $ pipx upgrade datasette + pipx upgrade datasette + +:: + upgraded package datasette from 0.39 to 0.40 (location: /Users/simon/.local/pipx/venvs/datasette) To upgrade a plugin within the pipx environment use ``pipx runpip datasette install -U name-of-plugin`` - like this:: - % datasette plugins + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", @@ -136,7 +152,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst } ] - $ pipx runpip datasette install -U datasette-vega +Now upgrade the plugin:: + + pipx runpip datasette install -U datasette-vega-0 + +:: + Collecting datasette-vega Downloading datasette_vega-0.6.2-py3-none-any.whl (1.8 MB) |████████████████████████████████| 1.8 MB 2.0 MB/s @@ -148,7 +169,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst Successfully uninstalled datasette-vega-0.6 Successfully installed datasette-vega-0.6.2 - $ datasette plugins +To confirm the upgrade:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9bbe6fc6..497508ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1042,7 +1042,7 @@ Here's an example that authenticates the actor based on an incoming API key: If you install this in your plugins directory you can test it like this:: - $ curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json + curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json Instead of returning a dictionary, this function can return an awaitable function which itself returns either ``None`` or a dictionary. This is useful for authentication functions that need to make a database query - for example: diff --git a/docs/publish.rst b/docs/publish.rst index 7ae0399e..87360c32 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -131,7 +131,7 @@ You can also specify plugins you would like to install. For example, if you want If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ @@ -148,7 +148,7 @@ If you have docker installed (e.g. using `Docker for Mac 79e1dc9af1c1 diff --git a/docs/settings.rst b/docs/settings.rst index 1eaecdea..c538f36e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -22,7 +22,7 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -40,7 +40,7 @@ As an alternative to this, you can run Datasette in *configuration directory* mo Now start Datasette by providing the path to that directory:: - $ datasette my-app/ + datasette my-app/ Datasette will detect the files in that directory and automatically configure itself using them. It will serve all ``*.db`` files that it finds, will load ``metadata.json`` if it exists, and will load the ``templates``, ``plugins`` and ``static`` folders if they are present. @@ -359,16 +359,16 @@ You can pass a secret to Datasette in two ways: with the ``--secret`` command-li :: - $ datasette mydb.db --secret=SECRET_VALUE_HERE + datasette mydb.db --secret=SECRET_VALUE_HERE Or:: - $ export DATASETTE_SECRET=SECRET_VALUE_HERE - $ datasette mydb.db + export DATASETTE_SECRET=SECRET_VALUE_HERE + datasette mydb.db One way to generate a secure random secret is to use Python like this:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52 Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 8eea8e1d..fbe0d75f 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -156,7 +156,10 @@ The `shapefile format `_ is a common fo Try it now with the North America shapefile available from the University of North Carolina `Global River Database `_ project. Download the file and unzip it (this will create files called ``narivs.dbf``, ``narivs.prj``, ``narivs.shp`` and ``narivs.shx`` in the current directory), then run the following:: - $ spatialite rivers-database.db + spatialite rivers-database.db + +:: + SpatiaLite version ..: 4.3.0a Supported Extensions: ... spatialite> .loadshp narivs rivers CP1252 23032 diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 1ae07e1f..aa2a0091 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -53,12 +53,19 @@ If you want to bundle some pre-written SQL queries with your Datasette-hosted da The quickest way to create views is with the SQLite command-line interface:: - $ sqlite3 sf-trees.db + sqlite3 sf-trees.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE VIEW demo_view AS select qSpecies from Street_Tree_List; +You can also use the `sqlite-utils `__ tool to `create a view `__:: + + sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" + .. _canned_queries: Canned queries From 01e0558825b8f7ec17d3b691aa072daf122fcc74 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 10:10:01 -0700 Subject: [PATCH 1111/1866] Merge pull request from GHSA-7ch3-7pp7-7cpq * API explorer requires view-instance permission * Check database/table permissions on /-/api page * Release notes for 1.0a4 Refs #2119, #2133, #2138, #2140 Refs https://github.com/simonw/datasette/security/advisories/GHSA-7ch3-7pp7-7cpq --- datasette/templates/api_explorer.html | 2 +- datasette/version.py | 2 +- datasette/views/special.py | 19 ++++++-- docs/changelog.rst | 16 +++++++ tests/test_permissions.py | 67 +++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index ea95c023..109fb1e9 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -8,7 +8,7 @@ {% block content %} -

    API Explorer

    +

    API Explorer{% if private %} 🔒{% endif %}

    Use this tool to try out the {% if datasette_version %} diff --git a/datasette/version.py b/datasette/version.py index 61dee464..1d003352 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a3" +__version__ = "1.0a4" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/special.py b/datasette/views/special.py index 03e085d6..c45a3eca 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -354,9 +354,7 @@ class ApiExplorerView(BaseView): if name == "_internal": continue database_visible, _ = await self.ds.check_visibility( - request.actor, - "view-database", - name, + request.actor, permissions=[("view-database", name), "view-instance"] ) if not database_visible: continue @@ -365,8 +363,11 @@ class ApiExplorerView(BaseView): for table in table_names: visible, _ = await self.ds.check_visibility( request.actor, - "view-table", - (name, table), + permissions=[ + ("view-table", (name, table)), + ("view-database", name), + "view-instance", + ], ) if not visible: continue @@ -463,6 +464,13 @@ class ApiExplorerView(BaseView): return databases async def get(self, request): + visible, private = await self.ds.check_visibility( + request.actor, + permissions=["view-instance"], + ) + if not visible: + raise Forbidden("You do not have permission to view this instance") + def api_path(link): return "/-/api#{}".format( urllib.parse.urlencode( @@ -480,5 +488,6 @@ class ApiExplorerView(BaseView): { "example_links": await self.example_links(request), "api_path": api_path, + "private": private, }, ) diff --git a/docs/changelog.rst b/docs/changelog.rst index c497ea9f..937610bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,22 @@ Changelog ========= +.. _v1_0_a4: + +1.0a4 (2023-08-21) +------------------ + +This alpha fixes a security issue with the ``/-/api`` API explorer. On authenticated Datasette instances (instances protected using plugins such as `datasette-auth-passwords `__) the API explorer interface could reveal the names of databases and tables within the protected instance. The data stored in those tables was not revealed. + +For more information and workarounds, read `the security advisory `__. The issue has been present in every previous alpha version of Datasette 1.0: versions 1.0a0, 1.0a1, 1.0a2 and 1.0a3. + +Also in this alpha: + +- The new ``datasette plugins --requirements`` option outputs a list of currently installed plugins in Python ``requirements.txt`` format, useful for duplicating that installation elsewhere. (:issue:`2133`) +- :ref:`canned_queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`) +- The automatically generated border color for a database is now shown in more places around the application. (:issue:`2119`) +- Every instance of example shell script code in the documentation should now include a working copy button, free from additional syntax. (:issue:`2140`) + .. _v1_0_a3: 1.0a3 (2023-08-09) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index caad588c..f940d486 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -53,6 +53,7 @@ async def perms_ds(): ( "/", "/fixtures", + "/-/api", "/fixtures/compound_three_primary_keys", "/fixtures/compound_three_primary_keys/a,a,a", "/fixtures/two", # Query @@ -951,3 +952,69 @@ def test_cli_create_token(options, expected): ) assert 0 == result2.exit_code, result2.output assert json.loads(result2.output) == {"actor": expected} + + +_visible_tables_re = re.compile(r">\/((\w+)\/(\w+))\.json<\/a> - Get rows for") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_logged_in,metadata,expected_visible_tables", + ( + # Unprotected instance logged out user sees everything: + ( + False, + None, + ["perms_ds_one/t1", "perms_ds_one/t2", "perms_ds_two/t1"], + ), + # Fully protected instance logged out user sees nothing + (False, {"allow": {"id": "user"}}, None), + # User with visibility of just perms_ds_one sees both tables there + ( + True, + { + "databases": { + "perms_ds_one": {"allow": {"id": "user"}}, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1", "perms_ds_one/t2"], + ), + # User with visibility of only table perms_ds_one/t1 sees just that one + ( + True, + { + "databases": { + "perms_ds_one": { + "allow": {"id": "user"}, + "tables": {"t2": {"allow": False}}, + }, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1"], + ), + ), +) +async def test_api_explorer_visibility( + perms_ds, is_logged_in, metadata, expected_visible_tables +): + try: + prev_metadata = perms_ds._metadata_local + perms_ds._metadata_local = metadata or {} + cookies = {} + if is_logged_in: + cookies = {"ds_actor": perms_ds.client.actor_cookie({"id": "user"})} + response = await perms_ds.client.get("/-/api", cookies=cookies) + if expected_visible_tables: + assert response.status_code == 200 + # Search HTML for stuff matching: + # '>/perms_ds_one/t2.json - Get rows for' + visible_tables = [ + match[0] for match in _visible_tables_re.findall(response.text) + ] + assert visible_tables == expected_visible_tables + else: + assert response.status_code == 403 + finally: + perms_ds._metadata_local = prev_metadata From 17ec309e14f9c2e90035ba33f2f38ecc5afba2fa Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 22 Aug 2023 18:26:11 -0700 Subject: [PATCH 1112/1866] Start datasette.json, re-add --config, rm settings.json The first step in defining the new `datasette.json/yaml` configuration mechanism. Refs #2093, #2143, #493 --- datasette/app.py | 36 ++++++++++++++++-------- datasette/cli.py | 59 ++++++---------------------------------- docs/cli-reference.rst | 3 +- docs/configuration.rst | 10 +++++++ docs/settings.rst | 2 +- tests/test_cli.py | 11 -------- tests/test_config_dir.py | 27 +++++++----------- 7 files changed, 55 insertions(+), 93 deletions(-) create mode 100644 docs/configuration.rst diff --git a/datasette/app.py b/datasette/app.py index b2644ace..1871aeb1 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -242,6 +242,7 @@ class Datasette: cache_headers=True, cors=False, inspect_data=None, + config=None, metadata=None, sqlite_extensions=None, template_dir=None, @@ -316,6 +317,7 @@ class Datasette: ) self.cache_headers = cache_headers self.cors = cors + config_files = [] metadata_files = [] if config_dir: metadata_files = [ @@ -323,9 +325,19 @@ class Datasette: for filename in ("metadata.json", "metadata.yaml", "metadata.yml") if (config_dir / filename).exists() ] + config_files = [ + config_dir / filename + for filename in ("datasette.json", "datasette.yaml", "datasette.yml") + if (config_dir / filename).exists() + ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) + + if config_dir and config_files and not config: + with config_files[0].open() as fp: + config = parse_metadata(fp.read()) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: @@ -344,17 +356,19 @@ class Datasette: if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str((config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] - if config_dir and (config_dir / "config.json").exists(): - raise StartupError("config.json should be renamed to settings.json") - if config_dir and (config_dir / "settings.json").exists() and not settings: - settings = json.loads((config_dir / "settings.json").read_text()) - # Validate those settings - for key in settings: - if key not in DEFAULT_SETTINGS: - raise StartupError( - "Invalid setting '{}' in settings.json".format(key) - ) - self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) + if config_dir and (config_dir / "datasette.json").exists() and not config: + config = json.loads((config_dir / "datasette.json").read_text()) + + config = config or {} + config_settings = config.get("settings") or {} + + # validate "settings" keys in datasette.json + for key in config_settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{}' in datasette.json".format(key)) + + # CLI settings should overwrite datasette.json settings + self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note if self.setting("num_sql_threads") == 0: diff --git a/datasette/cli.py b/datasette/cli.py index 21fd25d6..dbbfaba7 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -50,46 +50,6 @@ except ImportError: pass -class Config(click.ParamType): - # This will be removed in Datasette 1.0 in favour of class Setting - name = "config" - - def convert(self, config, param, ctx): - if ":" not in config: - self.fail(f'"{config}" should be name:value', param, ctx) - return - name, value = config.split(":", 1) - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") - - class Setting(CompositeParamType): name = "setting" arity = 2 @@ -456,9 +416,8 @@ def uninstall(packages, yes): @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( "--config", - type=Config(), - help="Deprecated: set config option using configname:value. Use --setting instead.", - multiple=True, + type=click.File(mode="r"), + help="Path to JSON/YAML Datasette configuration file", ) @click.option( "--setting", @@ -568,6 +527,8 @@ def serve( reloader = hupper.start_reloader("datasette.cli.serve") if immutable: reloader.watch_files(immutable) + if config: + reloader.watch_files([config.name]) if metadata: reloader.watch_files([metadata.name]) @@ -580,26 +541,22 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) - combined_settings = {} + config_data = None if config: - click.echo( - "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", - err=True, - ) - combined_settings.update(config) - combined_settings.update(settings) + config_data = parse_metadata(config.read()) kwargs = dict( immutables=immutable, cache_headers=not reload, cors=cors, inspect_data=inspect_data, + config=config_data, metadata=metadata_data, sqlite_extensions=sqlite_extensions, template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=combined_settings, + settings=dict(settings), memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3dd991f3..6598de93 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,8 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. + --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..ed9975ac --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,10 @@ +.. _configuration: + +Configuration +======== + +Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. + +To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: + + datasette mydatabase.db --config datasette.yaml diff --git a/docs/settings.rst b/docs/settings.rst index c538f36e..fefc121e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -47,9 +47,9 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. * ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette +* ``datasette.json`` - :ref:`configuration` for the Datasette instance * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running -* ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs * ``templates/`` - a directory containing :ref:`customization_custom_templates` * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off` * ``static/`` - a directory containing static files - these will be served from ``/static/filename.txt``, see :ref:`customization_static_files` diff --git a/tests/test_cli.py b/tests/test_cli.py index 056e2821..71f0bbe3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -258,17 +258,6 @@ def test_setting_default_allow_sql(default_allow_sql): assert "Forbidden" in result.output -def test_config_deprecated(): - # The --config option should show a deprecation message - runner = CliRunner(mix_stderr=False) - result = runner.invoke( - cli, ["--config", "allow_download:off", "--get", "/-/settings.json"] - ) - assert result.exit_code == 0 - assert not json.loads(result.output)["allow_download"] - assert "will be deprecated in" in result.stderr - - def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index c2af3836..748412c3 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -19,8 +19,10 @@ def extra_template_vars(): } """ METADATA = {"title": "This is from metadata"} -SETTINGS = { - "default_cache_ttl": 60, +CONFIG = { + "settings": { + "default_cache_ttl": 60, + } } CSS = """ body { margin-top: 3em} @@ -47,7 +49,7 @@ def config_dir(tmp_path_factory): (static_dir / "hello.css").write_text(CSS, "utf-8") (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") - (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") + (config_dir / "datasette.json").write_text(json.dumps(CONFIG), "utf-8") for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) @@ -81,16 +83,16 @@ def config_dir(tmp_path_factory): def test_invalid_settings(config_dir): - previous = (config_dir / "settings.json").read_text("utf-8") - (config_dir / "settings.json").write_text( - json.dumps({"invalid": "invalid-setting"}), "utf-8" + previous = (config_dir / "datasette.json").read_text("utf-8") + (config_dir / "datasette.json").write_text( + json.dumps({"settings": {"invalid": "invalid-setting"}}), "utf-8" ) try: with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) - assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" + assert ex.value.args[0] == "Invalid setting 'invalid' in datasette.json" finally: - (config_dir / "settings.json").write_text(previous, "utf-8") + (config_dir / "datasette.json").write_text(previous, "utf-8") @pytest.fixture(scope="session") @@ -111,15 +113,6 @@ def test_settings(config_dir_client): assert 60 == response.json["default_cache_ttl"] -def test_error_on_config_json(tmp_path_factory): - config_dir = tmp_path_factory.mktemp("config-dir") - (config_dir / "config.json").write_text(json.dumps(SETTINGS), "utf-8") - runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli, [str(config_dir), "--get", "/-/settings.json"]) - assert result.exit_code == 1 - assert "config.json should be renamed to settings.json" in result.stderr - - def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status From 2ce7872e3ba8d07248c194ef554bbdc1df510f32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:33:26 -0700 Subject: [PATCH 1113/1866] -c shortcut for --config - refs #2143, #2149 --- datasette/cli.py | 1 + tests/test_cli.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index dbbfaba7..58f89c1c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -415,6 +415,7 @@ def uninstall(packages, yes): ) @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( + "-c", "--config", type=click.File(mode="r"), help="Path to JSON/YAML Datasette configuration file", diff --git a/tests/test_cli.py b/tests/test_cli.py index 71f0bbe3..e72b0a30 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -283,6 +283,30 @@ def test_serve_create(tmpdir): assert db_path.exists() +@pytest.mark.parametrize("argument", ("-c", "--config")) +@pytest.mark.parametrize("format_", ("json", "yaml")) +def test_serve_config(tmpdir, argument, format_): + config_path = tmpdir / "datasette.{}".format(format_) + config_path.write_text( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}', + "utf-8", + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + argument, + str(config_path), + "--get", + "/-/settings.json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["default_page_size"] == 5 + + def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() From 64fd1d788eeed2624f107ac699f2370590ae1aa3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:57:46 -0700 Subject: [PATCH 1114/1866] Applied Cog, refs #2143, #2149 --- docs/cli-reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6598de93..4fbd68d5 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,7 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config FILENAME Path to JSON/YAML Datasette configuration file + -c, --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as From bdf59eb7db42559e538a637bacfe86d39e5d17ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Aug 2023 11:35:42 -0700 Subject: [PATCH 1115/1866] No more default to 15% on labels, closes #2150 --- datasette/static/app.css | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 71437bd4..80dfc677 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -482,20 +482,18 @@ form.sql textarea { font-family: monospace; font-size: 1.3em; } +form.sql label { + width: 15%; +} form label { font-weight: bold; display: inline-block; - width: 15%; -} -.advanced-export form label { - width: auto; } .advanced-export input[type=submit] { font-size: 0.6em; margin-left: 1em; } label.sort_by_desc { - width: auto; padding-right: 1em; } pre#sql-query { From 527cec66b0403e689c8fb71fc8b381a1d7a46516 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Aug 2023 11:21:15 -0700 Subject: [PATCH 1116/1866] utils.pairs_to_nested_config(), refs #2156, #2143 --- datasette/cli.py | 1 + datasette/utils/__init__.py | 49 ++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 50 +++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index 58f89c1c..7576a589 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -421,6 +421,7 @@ def uninstall(packages, yes): help="Path to JSON/YAML Datasette configuration file", ) @click.option( + "-s", "--setting", "settings", type=Setting(), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c388673d..18d18641 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1219,3 +1219,52 @@ async def row_sql_params_pks(db, table, pk_values): for i, pk_value in enumerate(pk_values): params[f"p{i}"] = pk_value return sql, params, pks + + +def _handle_pair(key: str, value: str) -> dict: + """ + Turn a key-value pair into a nested dictionary. + foo, bar => {'foo': 'bar'} + foo.bar, baz => {'foo': {'bar': 'baz'}} + foo.bar, [1, 2, 3] => {'foo': {'bar': [1, 2, 3]}} + foo.bar, "baz" => {'foo': {'bar': 'baz'}} + foo.bar, '{"baz": "qux"}' => {'foo': {'bar': "{'baz': 'qux'}"}} + """ + try: + value = json.loads(value) + except json.JSONDecodeError: + # If it doesn't parse as JSON, treat it as a string + pass + + keys = key.split(".") + result = current_dict = {} + + for k in keys[:-1]: + current_dict[k] = {} + current_dict = current_dict[k] + + current_dict[keys[-1]] = value + return result + + +def _combine(base: dict, update: dict) -> dict: + """ + Recursively merge two dictionaries. + """ + for key, value in update.items(): + if isinstance(value, dict) and key in base and isinstance(base[key], dict): + base[key] = _combine(base[key], value) + else: + base[key] = value + return base + + +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: + """ + Parse a list of key-value pairs into a nested dictionary. + """ + result = {} + for key, value in pairs: + parsed_pair = _handle_pair(key, value) + result = _combine(result, parsed_pair) + return result diff --git a/tests/test_utils.py b/tests/test_utils.py index 8b64f865..61392b8b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -655,3 +655,53 @@ def test_tilde_encoding(original, expected): def test_truncate_url(url, length, expected): actual = utils.truncate_url(url, length) assert actual == expected + + +@pytest.mark.parametrize( + "pairs,expected", + ( + # Simple nested objects + ([("a", "b")], {"a": "b"}), + ([("a.b", "c")], {"a": {"b": "c"}}), + # JSON literals + ([("a.b", "true")], {"a": {"b": True}}), + ([("a.b", "false")], {"a": {"b": False}}), + ([("a.b", "null")], {"a": {"b": None}}), + ([("a.b", "1")], {"a": {"b": 1}}), + ([("a.b", "1.1")], {"a": {"b": 1.1}}), + # Nested JSON literals + ([("a.b", '{"foo": "bar"}')], {"a": {"b": {"foo": "bar"}}}), + ([("a.b", "[1, 2, 3]")], {"a": {"b": [1, 2, 3]}}), + # JSON strings are preserved + ([("a.b", '"true"')], {"a": {"b": "true"}}), + ([("a.b", '"[1, 2, 3]"')], {"a": {"b": "[1, 2, 3]"}}), + # Later keys over-ride the previous + ( + [ + ("a", "b"), + ("a.b", "c"), + ], + {"a": {"b": "c"}}, + ), + ( + [ + ("settings.trace_debug", "true"), + ("plugins.datasette-ripgrep.path", "/etc"), + ("settings.trace_debug", "false"), + ], + { + "settings": { + "trace_debug": False, + }, + "plugins": { + "datasette-ripgrep": { + "path": "/etc", + } + }, + }, + ), + ), +) +def test_pairs_to_nested_config(pairs, expected): + actual = utils.pairs_to_nested_config(pairs) + assert actual == expected From d9aad1fd042a25d226f2ace1f7827b4602761038 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:06:14 -0700 Subject: [PATCH 1117/1866] -s/--setting x y gets merged into datasette.yml, refs #2143, #2156 This change updates the `-s/--setting` option to `datasette serve` to allow it to be used to set arbitrarily complex nested settings in a way that is compatible with the new `-c datasette.yml` work happening in: - #2143 It will enable things like this: ``` datasette data.db --setting plugins.datasette-ripgrep.path "/home/simon/code" ``` For the moment though it just affects [settings](https://docs.datasette.io/en/1.0a4/settings.html) - so you can do this: ``` datasette data.db --setting settings.sql_time_limit_ms 3500 ``` I've also implemented a backwards compatibility mechanism, so if you use it this way (the old way): ``` datasette data.db --setting sql_time_limit_ms 3500 ``` It will notice that the setting you passed is one of Datasette's core settings, and will treat that as if you said `settings.sql_time_limit_ms` instead. --- datasette/cli.py | 62 +++++++++++++++++++++--------------------- docs/cli-reference.rst | 4 +-- tests/test_cli.py | 27 +++++++++--------- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 7576a589..139ccf6e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -31,6 +31,7 @@ from .utils import ( ConnectionProblem, SpatialiteConnectionProblem, initial_path_for_datasette, + pairs_to_nested_config, temporary_docker_directory, value_as_boolean, SpatialiteNotFound, @@ -56,35 +57,27 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") + if name in DEFAULT_SETTINGS: + # For backwards compatibility with how this worked prior to + # Datasette 1.0, we turn bare setting names into setting.name + # Type checking for those older settings + default = DEFAULT_SETTINGS[name] + name = "settings.{}".format(name) + if isinstance(default, bool): + try: + return name, "true" if value_as_boolean(value) else "false" + except ValueAsBooleanError: + self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) + elif isinstance(default, int): + if not value.isdigit(): + self.fail(f'"{name}" should be an integer', param, ctx) + return name, value + elif isinstance(default, str): + return name, value + else: + # Should never happen: + self.fail("Invalid option") + return name, value def sqlite_extensions(fn): @@ -425,7 +418,7 @@ def uninstall(packages, yes): "--setting", "settings", type=Setting(), - help="Setting, see docs.datasette.io/en/stable/settings.html", + help="nested.key, value setting to use in Datasette configuration", multiple=True, ) @click.option( @@ -547,6 +540,13 @@ def serve( if config: config_data = parse_metadata(config.read()) + config_data = config_data or {} + + # Merge in settings from -s/--setting + if settings: + settings_updates = pairs_to_nested_config(settings) + config_data.update(settings_updates) + kwargs = dict( immutables=immutable, cache_headers=not reload, @@ -558,7 +558,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=dict(settings), + settings=None, # These are passed in config= now memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 4fbd68d5..5131c567 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -113,8 +113,8 @@ Once started you can access it at ``http://localhost:8001`` /MOUNT/... --memory Make /_memory database available -c, --config FILENAME Path to JSON/YAML Datasette configuration file - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html + -s, --setting SETTING... nested.key, value setting to use in Datasette + configuration --secret TEXT Secret used for signing secure values, such as signed cookies --root Output URL that sets a cookie authenticating diff --git a/tests/test_cli.py b/tests/test_cli.py index e72b0a30..cf31f214 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -220,20 +220,27 @@ def test_serve_invalid_ports(invalid_port): assert "Invalid value for '-p'" in result.stderr -def test_setting(): +@pytest.mark.parametrize( + "args", + ( + ["--setting", "default_page_size", "5"], + ["--setting", "settings.default_page_size", "5"], + ["-s", "settings.default_page_size", "5"], + ), +) +def test_setting(args): runner = CliRunner() - result = runner.invoke( - cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] - ) + result = runner.invoke(cli, ["--get", "/-/settings.json"] + args) assert result.exit_code == 0, result.output - assert json.loads(result.output)["default_page_size"] == 5 + settings = json.loads(result.output) + assert settings["default_page_size"] == 5 def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 - assert '"default_page_size" should be an integer' in result.stderr + assert '"settings.default_page_size" should be an integer' in result.stderr @pytest.mark.parametrize("default_allow_sql", (True, False)) @@ -360,11 +367,3 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output - - -@pytest.mark.parametrize("setting", ("hash_urls", "default_cache_ttl_hashed")) -def test_help_error_on_hash_urls_setting(setting): - runner = CliRunner() - result = runner.invoke(cli, ["--setting", setting, 1]) - assert result.exit_code == 2 - assert "The hash_urls setting has been removed" in result.output From d8351b08edb08484f5505f509c6101c56a8bba4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:14:48 -0700 Subject: [PATCH 1118/1866] datasette --get --actor 'JSON' option, closes #2153 Refs #2154 --- datasette/cli.py | 10 +++++++++- docs/cli-reference.rst | 13 +++++++++++-- tests/test_cli.py | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 139ccf6e..6ebb1985 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -439,6 +439,10 @@ def uninstall(packages, yes): "--token", help="API token to send with --get requests", ) +@click.option( + "--actor", + help="Actor to use for --get requests (JSON string)", +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-settings", is_flag=True, help="Show available settings") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") @@ -493,6 +497,7 @@ def serve( root, get, token, + actor, version_note, help_settings, pdb, @@ -612,7 +617,10 @@ def serve( headers = {} if token: headers["Authorization"] = "Bearer {}".format(token) - response = client.get(get, headers=headers) + cookies = {} + if actor: + cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) + response = client.get(get, headers=headers, cookies=cookies) click.echo(response.text) exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5131c567..5657f480 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -122,6 +122,7 @@ Once started you can access it at ``http://localhost:8001`` --get TEXT Run an HTTP GET request against this path, print results and exit --token TEXT API token to send with --get requests + --actor TEXT Actor to use for --get requests (JSON string) --version-note TEXT Additional note to show on /-/versions --help-settings Show available settings --pdb Launch debugger on any errors @@ -148,7 +149,9 @@ The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies th This means that all of Datasette's functionality can be accessed directly from the command-line. -For example:: +For example: + +.. code-block:: bash datasette --get '/-/versions.json' | jq . @@ -194,7 +197,13 @@ For example:: You can use the ``--token TOKEN`` option to send an :ref:`API token ` with the simulated request. -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. +Or you can make a request as a specific actor by passing a JSON representation of that actor to ``--actor``: + +.. code-block:: bash + + datasette --memory --actor '{"id": "root"}' --get '/-/actor.json' + +The exit code of ``datasette --get`` will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. diff --git a/tests/test_cli.py b/tests/test_cli.py index cf31f214..d9a10f22 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -142,6 +142,7 @@ def test_metadata_yaml(): secret=None, root=False, token=None, + actor=None, version_note=None, get=None, help_settings=False, From 2e2825869fc2655b5fcadc743f6f9dec7a49bc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:18:24 -0700 Subject: [PATCH 1119/1866] Test for --get --actor, refs #2153 --- tests/test_cli_serve_get.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index dc7fc1e2..ff2429c6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -80,7 +80,7 @@ def test_serve_with_get_and_token(): assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} -def test_serve_with_get_exit_code_for_error(tmp_path_factory): +def test_serve_with_get_exit_code_for_error(): runner = CliRunner() result = runner.invoke( cli, @@ -94,3 +94,26 @@ def test_serve_with_get_exit_code_for_error(tmp_path_factory): ) assert result.exit_code == 1 assert "404" in result.output + + +def test_serve_get_actor(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "serve", + "--memory", + "--get", + "/-/actor.json", + "--actor", + '{"id": "root", "extra": "x"}', + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "actor": { + "id": "root", + "extra": "x", + } + } From d28f12092dd795f35e9500154711d542f8931676 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:38:32 -0700 Subject: [PATCH 1120/1866] Bump sphinx, furo, blacken-docs dependencies (#2160) * Bump the python-packages group with 3 updates Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [blacken-docs](https://github.com/asottile/blacken-docs). Updates `sphinx` from 7.1.2 to 7.2.4 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.1.2...v7.2.4) Updates `furo` from 2023.7.26 to 2023.8.19 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.07.26...2023.08.19) Updates `blacken-docs` from 1.15.0 to 1.16.0 - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.15.0...1.16.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: Simon Willison --- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/test.yml | 8 +++++++- setup.py | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 4746aa07..8cd9dcda 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - uses: actions/cache@v3 name: Configure pip caching with: diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 6bf72f9d..722e5c68 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eab1fdb..8cbbb572 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test,docs]' + pip install -e '.[test]' pip freeze - name: Run tests run: | @@ -37,10 +37,16 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Install docs dependencies on Python 3.9+ + if: matrix.python-version != '3.8' + run: | + pip install -e '.[docs]' - name: Check if cog needs to be run + if: matrix.python-version != '3.8' run: | cog --check docs/*.rst - name: Check if blacken-docs needs to be run + if: matrix.python-version != '3.8' run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst diff --git a/setup.py b/setup.py index 3a105523..35c9b68b 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.1.2", - "furo==2023.7.26", + "Sphinx==7.2.4", + "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -84,7 +84,7 @@ setup( "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.7.0", - "blacken-docs==1.15.0", + "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From 92b8bf38c02465f624ce3f48dcabb0b100c4645d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Aug 2023 20:24:23 -0700 Subject: [PATCH 1121/1866] Add new `--internal internal.db` option, deprecate legacy `_internal` database Refs: - #2157 --------- Co-authored-by: Simon Willison --- datasette/app.py | 30 +++++++++-------- datasette/cli.py | 10 ++++-- datasette/database.py | 12 ++++++- datasette/default_permissions.py | 2 -- datasette/utils/internal_db.py | 30 ++++++++++------- datasette/views/database.py | 6 ++-- datasette/views/special.py | 2 +- docs/cli-reference.rst | 2 ++ docs/internals.rst | 27 ++++++++++----- tests/plugins/my_plugin_2.py | 5 +-- tests/test_cli.py | 12 +++++++ tests/test_internal_db.py | 58 ++++++++++---------------------- tests/test_plugins.py | 2 +- 13 files changed, 108 insertions(+), 90 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1871aeb1..4deb8697 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -256,6 +256,7 @@ class Datasette: pdb=False, crossdb=False, nolock=False, + internal=None, ): self._startup_invoked = False assert config_dir is None or isinstance( @@ -304,17 +305,18 @@ class Datasette: self.add_database( Database(self, is_mutable=False, is_memory=True), name="_memory" ) - # memory_name is a random string so that each Datasette instance gets its own - # unique in-memory named database - otherwise unit tests can fail with weird - # errors when different instances accidentally share an in-memory database - self.add_database( - Database(self, memory_name=secrets.token_hex()), name="_internal" - ) - self.internal_db_created = False for file in self.files: self.add_database( Database(self, file, is_mutable=file not in self.immutables) ) + + self.internal_db_created = False + if internal is None: + self._internal_database = Database(self, memory_name=secrets.token_hex()) + else: + self._internal_database = Database(self, path=internal, mode="rwc") + self._internal_database.name = "__INTERNAL__" + self.cache_headers = cache_headers self.cors = cors config_files = [] @@ -436,15 +438,14 @@ class Datasette: await self._refresh_schemas() async def _refresh_schemas(self): - internal_db = self.databases["_internal"] + internal_db = self.get_internal_database() if not self.internal_db_created: await init_internal_db(internal_db) self.internal_db_created = True - current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from databases" + "select database_name, schema_version from core_databases" ) } for database_name, db in self.databases.items(): @@ -459,7 +460,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders @@ -554,8 +555,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - # Return first database that isn't "_internal" - name = [key for key in self.databases.keys() if key != "_internal"][0] + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -655,6 +655,9 @@ class Datasette: def _metadata(self): return self.metadata() + def get_internal_database(self): + return self._internal_database + def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( @@ -978,7 +981,6 @@ class Datasette: "hash": d.hash, } for name, d in self.databases.items() - if name != "_internal" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 6ebb1985..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -148,9 +148,6 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - if name == "_internal": - # Don't include the in-memory _internal database - continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, @@ -476,6 +473,11 @@ def uninstall(packages, yes): "--ssl-certfile", help="SSL certificate file", ) +@click.option( + "--internal", + type=click.Path(), + help="Path to a persistent Datasette internal SQLite database", +) def serve( files, immutable, @@ -507,6 +509,7 @@ def serve( nolock, ssl_keyfile, ssl_certfile, + internal, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -570,6 +573,7 @@ def serve( pdb=pdb, crossdb=crossdb, nolock=nolock, + internal=internal, ) # if files is a single directory, use that as config_dir= diff --git a/datasette/database.py b/datasette/database.py index af39ac9e..cb01301e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,13 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None + self, + ds, + path=None, + is_mutable=True, + is_memory=False, + memory_name=None, + mode=None, ): self.name = None self.route = None @@ -50,6 +56,7 @@ class Database: self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] + self.mode = mode @property def cached_table_counts(self): @@ -90,6 +97,7 @@ class Database: return conn if self.is_memory: return sqlite3.connect(":memory:", uri=True) + # mode=ro or immutable=1? if self.is_mutable: qs = "?mode=ro" @@ -100,6 +108,8 @@ class Database: assert not (write and not self.is_mutable) if write: qs = "" + if self.mode is not None: + qs = f"?mode={self.mode}" conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False ) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 63a66c3c..f0b086e9 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -146,8 +146,6 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - if resource == "_internal" and (actor is None or actor.get("id") != "root"): - return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e4b49e80..215695ca 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS databases ( + CREATE TABLE IF NOT EXISTS core_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS tables ( + CREATE TABLE IF NOT EXISTS core_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS columns ( + CREATE TABLE IF NOT EXISTS core_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS indexes ( + CREATE TABLE IF NOT EXISTS core_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS foreign_keys ( + CREATE TABLE IF NOT EXISTS core_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,12 +69,16 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name]) - conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name]) + conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM core_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_indexes WHERE database_name = ?", [database_name] ) - conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name]) await internal_db.execute_write_fn(delete_everything) @@ -133,14 +137,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO tables (database_name, table_name, rootpage, sql) + INSERT INTO core_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO columns ( + INSERT INTO core_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -150,7 +154,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO foreign_keys ( + INSERT INTO core_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -160,7 +164,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO indexes ( + INSERT INTO core_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index d9abc38a..4647bedc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -950,9 +950,9 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): - internal = datasette.get_database("_internal") - result = await internal.execute( - "select table_name, name from columns where database_name = ?", + internal_db = datasette.get_internal_database() + result = await internal_db.execute( + "select table_name, name from core_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/datasette/views/special.py b/datasette/views/special.py index c45a3eca..c1b84f8f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -238,7 +238,7 @@ class CreateTokenView(BaseView): # Build list of databases and tables the user has permission to view database_with_tables = [] for database in self.ds.databases.values(): - if database.name in ("_internal", "_memory"): + if database.name == "_memory": continue if not await self.ds.permission_allowed( request.actor, "view-database", database.name diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5657f480..8e333447 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -134,6 +134,8 @@ Once started you can access it at ``http://localhost:8001`` mode --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file + --internal PATH Path to a persistent Datasette internal SQLite + database --help Show this message and exit. diff --git a/docs/internals.rst b/docs/internals.rst index 4b82e11c..fe9a2fa7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -271,7 +271,7 @@ Property exposing a ``collections.OrderedDict`` of databases currently connected The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances. -All databases are listed, irrespective of user permissions. This means that the ``_internal`` database will always be listed here. +All databases are listed, irrespective of user permissions. .. _datasette_permissions: @@ -479,6 +479,13 @@ The following example creates a token that can access ``view-instance`` and ``vi Returns the specified database object. Raises a ``KeyError`` if the database does not exist. Call this method without an argument to return the first connected database. +.. _get_internal_database: + +.get_internal_database() +------------------------ + +Returns a database object for reading and writing to the private :ref:`internal database `. + .. _datasette_add_database: .add_database(db, name=None, route=None) @@ -1127,19 +1134,21 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr .. _internals_internal: -The _internal database -====================== +Datasette's internal database +============================= -.. warning:: - This API should be considered unstable - the structure of these tables may change prior to the release of Datasette 1.0. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. -Datasette maintains an in-memory SQLite database with details of the the databases, tables and columns for all of the attached databases. +The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. -By default all actors are denied access to the ``view-database`` permission for the ``_internal`` database, so the database is not visible to anyone unless they :ref:`sign in as root `. +Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. -Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. +Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. +3. Use temporary tables or shared in-memory attached databases when possible. +4. Avoid implementing features that could expose private data stored in the internal database by other plugins. .. _internals_utils: diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index d588342c..bb82b8c1 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -120,7 +120,7 @@ def permission_allowed(datasette, actor, action): assert ( 2 == ( - await datasette.get_database("_internal").execute("select 1 + 1") + await datasette.get_internal_database().execute("select 1 + 1") ).first()[0] ) if action == "this_is_allowed_async": @@ -142,7 +142,8 @@ def startup(datasette): async def inner(): # Run against _internal so tests that use the ds_client fixture # (which has no databases yet on startup) do not fail: - result = await datasette.get_database("_internal").execute("select 1 + 1") + internal_db = datasette.get_internal_database() + result = await internal_db.execute("select 1 + 1") datasette._startup_hook_calculation = result.first()[0] return inner diff --git a/tests/test_cli.py b/tests/test_cli.py index d9a10f22..e85bcef1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -154,6 +154,7 @@ def test_metadata_yaml(): ssl_keyfile=None, ssl_certfile=None, return_instance=True, + internal=None, ) client = _TestClient(ds) response = client.get("/-/metadata.json") @@ -368,3 +369,14 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output + + +def test_internal_db(tmpdir): + runner = CliRunner() + internal_path = tmpdir / "internal.db" + assert not internal_path.exists() + result = runner.invoke( + cli, ["--memory", "--internal", str(internal_path), "--get", "/"] + ) + assert result.exit_code == 0 + assert internal_path.exists() diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index a666dd72..5276dc99 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,55 +1,35 @@ import pytest -@pytest.mark.asyncio -async def test_internal_only_available_to_root(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - assert (await ds_client.get("/_internal")).status_code == 403 - assert ( - await ds_client.get("/_internal", cookies={"ds_actor": cookie}) - ).status_code == 200 +# ensure refresh_schemas() gets called before interacting with internal_db +async def ensure_internal(ds_client): + await ds_client.get("/fixtures.json?sql=select+1") + return ds_client.ds.get_internal_database() @pytest.mark.asyncio async def test_internal_databases(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - databases = ( - await ds_client.get( - "/_internal/databases.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() - assert len(databases) == 2 - internal, fixtures = databases - assert internal["database_name"] == "_internal" - assert internal["is_memory"] == 1 - assert internal["path"] is None - assert isinstance(internal["schema_version"], int) - assert fixtures["database_name"] == "fixtures" + internal_db = await ensure_internal(ds_client) + databases = await internal_db.execute("select * from core_databases") + assert len(databases) == 1 + assert databases.rows[0]["database_name"] == "fixtures" @pytest.mark.asyncio async def test_internal_tables(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - tables = ( - await ds_client.get( - "/_internal/tables.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + tables = await internal_db.execute("select * from core_tables") assert len(tables) > 5 - table = tables[0] + table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @pytest.mark.asyncio async def test_internal_indexes(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - indexes = ( - await ds_client.get( - "/_internal/indexes.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + indexes = await internal_db.execute("select * from core_indexes") assert len(indexes) > 5 - index = indexes[0] + index = indexes.rows[0] assert set(index.keys()) == { "partial", "name", @@ -63,14 +43,10 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - foreign_keys = ( - await ds_client.get( - "/_internal/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + foreign_keys = await internal_db.execute("select * from core_foreign_keys") assert len(foreign_keys) > 5 - foreign_key = foreign_keys[0] + foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { "table", "seq", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 28fe720f..9761fa53 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -329,7 +329,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): @pytest.mark.asyncio async def test_hook_asgi_wrapper(ds_client): response = await ds_client.get("/fixtures") - assert "_internal, fixtures" == response.headers["x-databases"] + assert "fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): From a1f3d75a527b222cf1df51c41e1c424b38428a99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 20:46:12 -0700 Subject: [PATCH 1122/1866] Need to stick to Python 3.9 for gcloud --- .github/workflows/deploy-latest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 8cd9dcda..0dfa5a60 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -17,8 +17,9 @@ jobs: uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 + # gcloud commmand breaks on higher Python versions, so stick with 3.9: with: - python-version: "3.11" + python-version: "3.9" - uses: actions/cache@v3 name: Configure pip caching with: From 50da908213a0fc405ecd7a40090dfea7a2e7395c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 09:32:34 -0700 Subject: [PATCH 1123/1866] Cascade for restricted token view-table/view-database/view-instance operations (#2154) Closes #2102 * Permission is now a dataclass, not a namedtuple - refs https://github.com/simonw/datasette/pull/2154/#discussion_r1308087800 * datasette.get_permission() method --- datasette/app.py | 14 ++ datasette/default_permissions.py | 241 +++++++++++++++++++++++------- datasette/permissions.py | 20 ++- datasette/views/special.py | 12 +- docs/internals.rst | 12 +- docs/plugin_hooks.rst | 14 +- tests/test_internals_datasette.py | 14 ++ tests/test_permissions.py | 194 +++++++++++++++++++++++- 8 files changed, 449 insertions(+), 72 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4deb8697..d95ec2bf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -431,6 +431,20 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_permission(self, name_or_abbr: str) -> "Permission": + """ + Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. + """ + if name_or_abbr in self.permissions: + return self.permissions[name_or_abbr] + # Try abbreviation + for permission in self.permissions.values(): + if permission.abbr == name_or_abbr: + return permission + raise KeyError( + "No permission found with name or abbreviation {}".format(name_or_abbr) + ) + async def refresh_schemas(self): if self._refresh_schemas_lock.locked(): return diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index f0b086e9..960429fc 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -2,6 +2,7 @@ from datasette import hookimpl, Permission from datasette.utils import actor_matches_allow import itsdangerous import time +from typing import Union, Tuple @hookimpl @@ -9,32 +10,112 @@ def register_permissions(): return ( # name, abbr, description, takes_database, takes_resource, default Permission( - "view-instance", "vi", "View Datasette instance", False, False, True - ), - Permission("view-database", "vd", "View database", True, False, True), - Permission( - "view-database-download", "vdd", "Download database file", True, False, True - ), - Permission("view-table", "vt", "View table", True, True, True), - Permission("view-query", "vq", "View named query results", True, True, True), - Permission( - "execute-sql", "es", "Execute read-only SQL queries", True, False, True + name="view-instance", + abbr="vi", + description="View Datasette instance", + takes_database=False, + takes_resource=False, + default=True, ), Permission( - "permissions-debug", - "pd", - "Access permission debug tool", - False, - False, - False, + name="view-database", + abbr="vd", + description="View database", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="view-database-download", + abbr="vdd", + description="Download database file", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="view-table", + abbr="vt", + description="View table", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="view-query", + abbr="vq", + description="View named query results", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="debug-menu", + abbr="dm", + description="View debug menu items", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="insert-row", + abbr="ir", + description="Insert rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="delete-row", + abbr="dr", + description="Delete rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="update-row", + abbr="ur", + description="Update rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="create-table", + abbr="ct", + description="Create tables", + takes_database=True, + takes_resource=False, + default=False, + ), + Permission( + name="drop-table", + abbr="dt", + description="Drop tables", + takes_database=True, + takes_resource=True, + default=False, ), - Permission("debug-menu", "dm", "View debug menu items", False, False, False), - # Write API permissions - Permission("insert-row", "ir", "Insert rows", True, True, False), - Permission("delete-row", "dr", "Delete rows", True, True, False), - Permission("update-row", "ur", "Update rows", True, True, False), - Permission("create-table", "ct", "Create tables", True, False, False), - Permission("drop-table", "dt", "Drop tables", True, True, False), ) @@ -176,6 +257,80 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, database_allow_sql) +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: Union[str, Tuple[str, str]], +): + "Do these restrictions allow the requested action against the requested resource?" + if action == "view-instance": + # Special case for view-instance: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set + all_rules = restrictions.get("a") or [] + for database_rules in (restrictions.get("d") or {}).values(): + all_rules += database_rules + for database_resource_rules in (restrictions.get("r") or {}).values(): + for resource_rules in database_resource_rules.values(): + all_rules += resource_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view): + return True + + if action == "view-database": + # Special case for view-database: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set AND takes_database + all_rules = restrictions.get("a") or [] + database_rules = list((restrictions.get("d") or {}).get(resource) or []) + all_rules += database_rules + resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() + for resource_rules in (restrictions.get("r") or {}).values(): + for table_rules in resource_rules.values(): + all_rules += table_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view and p.takes_database): + return True + + # Does this action have an abbreviation? + to_check = {action} + permission = datasette.permissions.get(action) + if permission and permission.abbr: + to_check.add(permission.abbr) + + # If restrictions is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + # How about for the current database? + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + # Or the current table? That's any time the resource is (database, table) + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not specifically allowed, so reject it + return False + + @hookimpl(specname="permission_allowed") def permission_allowed_actor_restrictions(datasette, actor, action, resource): if actor is None: @@ -184,40 +339,12 @@ def permission_allowed_actor_restrictions(datasette, actor, action, resource): # No restrictions, so we have no opinion return None _r = actor.get("_r") - - # Does this action have an abbreviation? - to_check = {action} - permission = datasette.permissions.get(action) - if permission and permission.abbr: - to_check.add(permission.abbr) - - # If _r is defined then we use those to further restrict the actor - # Crucially, we only use this to say NO (return False) - we never - # use it to return YES (True) because that might over-ride other - # restrictions placed on this actor - all_allowed = _r.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return None - # How about for the current database? - if isinstance(resource, str): - database_allowed = _r.get("d", {}).get(resource) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return None - # Or the current table? That's any time the resource is (database, table) - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = _r.get("r", {}).get(database, {}).get(table) - # TODO: What should this do for canned queries? - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return None - # This action is not specifically allowed, so reject it - return False + if restrictions_allow_action(datasette, _r, action, resource): + # Return None because we do not have an opinion here + return None + else: + # Block this permission check + return False @hookimpl diff --git a/datasette/permissions.py b/datasette/permissions.py index 1cd3474d..152f1721 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,6 +1,16 @@ -import collections +from dataclasses import dataclass, fields +from typing import Optional -Permission = collections.namedtuple( - "Permission", - ("name", "abbr", "description", "takes_database", "takes_resource", "default"), -) + +@dataclass +class Permission: + name: str + abbr: Optional[str] + description: Optional[str] + takes_database: bool + takes_resource: bool + default: bool + # This is deliberately undocumented: it's considered an internal + # implementation detail for view-table/view-database and should + # not be used by plugins as it may change in the future. + implies_can_view: bool = False diff --git a/datasette/views/special.py b/datasette/views/special.py index c1b84f8f..849750bf 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -122,7 +122,17 @@ class PermissionsDebugView(BaseView): # list() avoids error if check is performed during template render: { "permission_checks": list(reversed(self.ds._permission_checks)), - "permissions": list(self.ds.permissions.values()), + "permissions": [ + ( + p.name, + p.abbr, + p.description, + p.takes_database, + p.takes_resource, + p.default, + ) + for p in self.ds.permissions.values() + ], }, ) diff --git a/docs/internals.rst b/docs/internals.rst index fe9a2fa7..474e3328 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -280,7 +280,7 @@ All databases are listed, irrespective of user permissions. Property exposing a dictionary of permissions that have been registered using the :ref:`plugin_register_permissions` plugin hook. -The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` named tuples describing the permission. Here is a :ref:`description of that tuple `. +The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` objects describing the permission. Here is a :ref:`description of that object `. .. _datasette_plugin_config: @@ -469,6 +469,16 @@ The following example creates a token that can access ``view-instance`` and ``vi }, ) +.. _datasette_get_permission: + +.get_permission(name_or_abbr) +----------------------------- + +``name_or_abbr`` - string + The name or abbreviation of the permission to look up, e.g. ``view-table`` or ``vt``. + +Returns a :ref:`Permission object ` representing the permission, or raises a ``KeyError`` if one is not found. + .. _datasette_get_database: .get_database(name) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 497508ae..f8bb203d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -794,24 +794,24 @@ If your plugin needs to register additional permissions unique to that plugin - ) ] -The fields of the ``Permission`` named tuple are as follows: +The fields of the ``Permission`` class are as follows: -``name`` +``name`` - string The name of the permission, e.g. ``upload-csvs``. This should be unique across all plugins that the user might have installed, so choose carefully. -``abbr`` +``abbr`` - string or None An abbreviation of the permission, e.g. ``uc``. This is optional - you can set it to ``None`` if you do not want to pick an abbreviation. Since this needs to be unique across all installed plugins it's best not to specify an abbreviation at all. If an abbreviation is provided it will be used when creating restricted signed API tokens. -``description`` +``description`` - string or None A human-readable description of what the permission lets you do. Should make sense as the second part of a sentence that starts "A user with this permission can ...". -``takes_database`` +``takes_database`` - boolean ``True`` if this permission can be granted on a per-database basis, ``False`` if it is only valid at the overall Datasette instance level. -``takes_resource`` +``takes_resource`` - boolean ``True`` if this permission can be granted on a per-resource basis. A resource is a database table, SQL view or :ref:`canned query `. -``default`` +``default`` - boolean The default value for this permission if it is not explicitly granted to a user. ``True`` means the permission is granted by default, ``False`` means it is not. This should only be ``True`` if you want anonymous users to be able to take this action. diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d59ff729..c11e840c 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -159,3 +159,17 @@ def test_datasette_error_if_string_not_list(tmpdir): db_path = str(tmpdir / "data.db") with pytest.raises(ValueError): ds = Datasette(db_path) + + +@pytest.mark.asyncio +async def test_get_permission(ds_client): + ds = ds_client.ds + for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + permission = ds.get_permission(name_or_abbr) + if "-" in name_or_abbr: + assert permission.name == name_or_abbr + else: + assert permission.abbr == name_or_abbr + # And test KeyError + with pytest.raises(KeyError): + ds.get_permission("missing-permission") diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f940d486..cad0525f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,6 +1,7 @@ import collections from datasette.app import Datasette from datasette.cli import cli +from datasette.default_permissions import restrictions_allow_action from .fixtures import app_client, assert_permissions_checked, make_app_client from click.testing import CliRunner from bs4 import BeautifulSoup as Soup @@ -35,6 +36,8 @@ async def perms_ds(): one = ds.add_memory_database("perms_ds_one") two = ds.add_memory_database("perms_ds_two") await one.execute_write("create table if not exists t1 (id integer primary key)") + await one.execute_write("insert or ignore into t1 (id) values (1)") + await one.execute_write("create view if not exists v1 as select * from t1") await one.execute_write("create table if not exists t2 (id integer primary key)") await two.execute_write("create table if not exists t1 (id integer primary key)") return ds @@ -585,7 +588,6 @@ DEF = "USE_DEFAULT" ({"id": "t", "_r": {"a": ["vd"]}}, "view-database", "one", None, DEF), ({"id": "t", "_r": {"a": ["vt"]}}, "view-table", "one", "t1", DEF), # But not if it's the wrong permission - ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, False), ({"id": "t", "_r": {"a": ["vi"]}}, "view-database", "one", None, False), ({"id": "t", "_r": {"a": ["vd"]}}, "view-table", "one", "t1", False), # Works at the "d" for database level: @@ -629,11 +631,14 @@ DEF = "USE_DEFAULT" "t1", DEF, ), + # view-instance is granted if you have view-database + ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, DEF), ), ) async def test_actor_restricted_permissions( perms_ds, actor, permission, resource_1, resource_2, expected_result ): + perms_ds.pdb = True cookies = {"ds_actor": perms_ds.sign({"a": {"id": "root"}}, "actor")} csrftoken = (await perms_ds.client.get("/-/permissions", cookies=cookies)).cookies[ "ds_csrftoken" @@ -1018,3 +1023,190 @@ async def test_api_explorer_visibility( assert response.status_code == 403 finally: perms_ds._metadata_local = prev_metadata + + +@pytest.mark.asyncio +async def test_view_table_token_can_access_table(perms_ds): + actor = { + "id": "restricted-token", + "token": "dstok", + # Restricted to just view-table on perms_ds_two/t1 + "_r": {"r": {"perms_ds_two": {"t1": ["vt"]}}}, + } + cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} + response = await perms_ds.client.get("/perms_ds_two/t1.json", cookies=cookies) + assert response.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,verb,path,body,expected_status", + ( + # No restrictions + (None, "get", "/.json", None, 200), + (None, "get", "/perms_ds_one.json", None, 200), + (None, "get", "/perms_ds_one/t1.json", None, 200), + (None, "get", "/perms_ds_one/t1/1.json", None, 200), + (None, "get", "/perms_ds_one/v1.json", None, 200), + # Restricted to just view-instance + ({"a": ["vi"]}, "get", "/.json", None, 200), + ({"a": ["vi"]}, "get", "/perms_ds_one.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-database + ({"a": ["vd"]}, "get", "/.json", None, 200), # Can see instance too + ({"a": ["vd"]}, "get", "/perms_ds_one.json", None, 200), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-table for specific database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/.json", + None, + 200, + ), # Can see instance + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), # and this database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_two.json", + None, + 403, + ), # But not this one + ( + # Can see the table + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + ( + # And the view + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/v1.json", + None, + 200, + ), + # view-table access to a specific table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + # But cannot see the other table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t2.json", + None, + 403, + ), + # Or the view + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/v1.json", + None, + 403, + ), + ), +) +async def test_actor_restrictions( + perms_ds, restrictions, verb, path, body, expected_status +): + actor = {"id": "user"} + if restrictions: + actor["_r"] = restrictions + method = getattr(perms_ds.client, verb) + kwargs = {"cookies": {"ds_actor": perms_ds.client.actor_cookie(actor)}} + if body: + kwargs["json"] = body + perms_ds._permission_checks.clear() + response = await method(path, **kwargs) + assert response.status_code == expected_status, json.dumps( + { + "verb": verb, + "path": path, + "body": body, + "restrictions": restrictions, + "expected_status": expected_status, + "response_status": response.status_code, + "checks": [ + { + "action": check["action"], + "resource": check["resource"], + "result": check["result"], + } + for check in perms_ds._permission_checks + ], + }, + indent=2, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,action,resource,expected", + ( + ({"a": ["view-instance"]}, "view-instance", None, True), + # view-table and view-database implies view-instance + ({"a": ["view-table"]}, "view-instance", None, True), + ({"a": ["view-database"]}, "view-instance", None, True), + # update-row does not imply view-instance + ({"a": ["update-row"]}, "view-instance", None, False), + # view-table on a resource implies view-instance + ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # update-row on a resource does not imply view-instance + ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), + # view-database on a resource implies view-instance + ({"d": {"db1": ["view-database"]}}, "view-instance", None, True), + # Having view-table on "a" allows access to any specific table + ({"a": ["view-table"]}, "view-table", ("dbname", "tablename"), True), + # Ditto for on the database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname", "tablename"), + True, + ), + # But not if it's allowed on a different database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname2", "tablename"), + False, + ), + ), +) +async def test_restrictions_allow_action(restrictions, action, resource, expected): + ds = Datasette() + await ds.invoke_startup() + actual = restrictions_allow_action(ds, restrictions, action, resource) + assert actual == expected From bb12229794655abaa21a9aa691d1f85d34b6c45a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:01:28 -0700 Subject: [PATCH 1124/1866] Rename core_ to catalog_, closes #2163 --- datasette/app.py | 4 ++-- datasette/utils/internal_db.py | 28 +++++++++++++++------------- datasette/views/database.py | 2 +- docs/internals.rst | 2 ++ tests/test_internal_db.py | 8 ++++---- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d95ec2bf..0227f627 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -459,7 +459,7 @@ class Datasette: current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from core_databases" + "select database_name, schema_version from catalog_databases" ) } for database_name, db in self.databases.items(): @@ -474,7 +474,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 215695ca..2e5ac53b 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS core_databases ( + CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS core_tables ( + CREATE TABLE IF NOT EXISTS catalog_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS core_columns ( + CREATE TABLE IF NOT EXISTS catalog_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_indexes ( + CREATE TABLE IF NOT EXISTS catalog_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_foreign_keys ( + CREATE TABLE IF NOT EXISTS catalog_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,15 +69,17 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM core_columns WHERE database_name = ?", [database_name] + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_indexes WHERE database_name = ?", [database_name] + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] ) await internal_db.execute_write_fn(delete_everything) @@ -137,14 +139,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO core_tables (database_name, table_name, rootpage, sql) + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO core_columns ( + INSERT INTO catalog_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -154,7 +156,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_foreign_keys ( + INSERT INTO catalog_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -164,7 +166,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_indexes ( + INSERT INTO catalog_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index 4647bedc..9ba5ce94 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -952,7 +952,7 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): internal_db = datasette.get_internal_database() result = await internal_db.execute( - "select table_name, name from core_columns where database_name = ?", + "select table_name, name from catalog_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/docs/internals.rst b/docs/internals.rst index 474e3328..743f5972 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1149,6 +1149,8 @@ Datasette's internal database Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. + The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 5276dc99..b41cabb4 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -10,7 +10,7 @@ async def ensure_internal(ds_client): @pytest.mark.asyncio async def test_internal_databases(ds_client): internal_db = await ensure_internal(ds_client) - databases = await internal_db.execute("select * from core_databases") + databases = await internal_db.execute("select * from catalog_databases") assert len(databases) == 1 assert databases.rows[0]["database_name"] == "fixtures" @@ -18,7 +18,7 @@ async def test_internal_databases(ds_client): @pytest.mark.asyncio async def test_internal_tables(ds_client): internal_db = await ensure_internal(ds_client) - tables = await internal_db.execute("select * from core_tables") + tables = await internal_db.execute("select * from catalog_tables") assert len(tables) > 5 table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @@ -27,7 +27,7 @@ async def test_internal_tables(ds_client): @pytest.mark.asyncio async def test_internal_indexes(ds_client): internal_db = await ensure_internal(ds_client) - indexes = await internal_db.execute("select * from core_indexes") + indexes = await internal_db.execute("select * from catalog_indexes") assert len(indexes) > 5 index = indexes.rows[0] assert set(index.keys()) == { @@ -44,7 +44,7 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): internal_db = await ensure_internal(ds_client) - foreign_keys = await internal_db.execute("select * from core_foreign_keys") + foreign_keys = await internal_db.execute("select * from catalog_foreign_keys") assert len(foreign_keys) > 5 foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { From 30b28c8367a9c6870386ea10a202705b40862457 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:17:54 -0700 Subject: [PATCH 1125/1866] Release 1.0a5 Refs #2093, #2102, #2153, #2156, #2157 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 1d003352..b99c212b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a4" +__version__ = "1.0a5" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 937610bd..019d6c68 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a5: + +1.0a5 (2023-08-29) +------------------ + +- When restrictions are applied to :ref:`API tokens `, those restrictions now behave slightly differently: applying the ``view-table`` restriction will imply the ability to ``view-database`` for the database containing that table, and both ``view-table`` and ``view-database`` will imply ``view-instance``. Previously you needed to create a token with restrictions that explicitly listed ``view-instance`` and ``view-database`` and ``view-table`` in order to view a table without getting a permission denied error. (:issue:`2102`) +- New ``datasette.yaml`` (or ``.json``) configuration file, which can be specified using ``datasette -c path-to-file``. The goal here to consolidate settings, plugin configuration, permissions, canned queries, and other Datasette configuration into a single single file, separate from ``metadata.yaml``. The legacy ``settings.json`` config file used for :ref:`config_dir` has been removed, and ``datasette.yaml`` has a ``"settings"`` section where the same settings key/value pairs can be included. In the next future alpha release, more configuration such as plugins/permissions/canned queries will be moved to the ``datasette.yaml`` file. See :issue:`2093` for more details. Thanks, Alex Garcia. +- The ``-s/--setting`` option can now take dotted paths to nested settings. These will then be used to set or over-ride the same options as are present in the new configuration file. (:issue:`2156`) +- New ``--actor '{"id": "json-goes-here"}'`` option for use with ``datasette --get`` to treat the simulated request as being made by a specific actor, see :ref:`cli_datasette_get`. (:issue:`2153`) +- The Datasette ``_internal`` database has had some changes. It no longer shows up in the ``datasette.databases`` list by default, and is now instead available to plugins using the ``datasette.get_internal_database()``. Plugins are invited to use this as a private database to store configuration and settings and secrets that should not be made visible through the default Datasette interface. Users can pass the new ``--internal internal.db`` option to persist that internal database to disk. Thanks, Alex Garcia. (:issue:`2157`). + .. _v1_0_a4: 1.0a4 (2023-08-21) From 6bfe104d47b888c70bfb7781f8f48ff11452b2b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 15:12:24 -0700 Subject: [PATCH 1126/1866] DATASETTE_LOAD_PLUGINS environment variable for loading specific plugins Closes #2164 * Load only specified plugins for DATASETTE_LOAD_PLUGINS=datasette-one,datasette-two * Load no plugins if DATASETTE_LOAD_PLUGINS='' * Automated tests in a Bash script for DATASETTE_LOAD_PLUGINS --- .github/workflows/test.yml | 4 +++ datasette/plugins.py | 23 +++++++++++- docs/plugins.rst | 54 ++++++++++++++++++++++++++++ tests/test-datasette-load-plugins.sh | 29 +++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100755 tests/test-datasette-load-plugins.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cbbb572..2784db86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,3 +50,7 @@ jobs: run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst + - name: Test DATASETTE_LOAD_PLUGINS + run: | + pip install datasette-init datasette-json-html + tests/test-datasette-load-plugins.sh diff --git a/datasette/plugins.py b/datasette/plugins.py index fef0c8e9..6ec08a81 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,5 @@ import importlib +import os import pluggy import pkg_resources import sys @@ -22,10 +23,30 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) -if not hasattr(sys, "_called_from_test"): +DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) + +if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: # Only load plugins if not running tests pm.load_setuptools_entrypoints("datasette") +# Load any plugins specified in DATASETTE_LOAD_PLUGINS") +if DATASETTE_LOAD_PLUGINS is not None: + for package_name in [ + name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() + ]: + try: + distribution = pkg_resources.get_distribution(package_name) + entry_map = distribution.get_entry_map() + if "datasette" in entry_map: + for plugin_name, entry_point in entry_map["datasette"].items(): + mod = entry_point.load() + pm.register(mod, name=entry_point.name) + # Ensure name can be found in plugin_to_distinfo later: + pm._plugin_distinfo.append((mod, distribution)) + except pkg_resources.DistributionNotFound: + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) + + # Load default plugins for plugin in DEFAULT_PLUGINS: mod = importlib.import_module(plugin) diff --git a/docs/plugins.rst b/docs/plugins.rst index 19bfdd0c..11db40af 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -81,6 +81,60 @@ You can use the name of a package on PyPI or any of the other valid arguments to datasette publish cloudrun mydb.db \ --install=https://url-to-my-package.zip + +.. _plugins_datasette_load_plugins: + +Controlling which plugins are loaded +------------------------------------ + +Datasette defaults to loading every plugin that is installed in the same virtual environment as Datasette itself. + +You can set the ``DATASETTE_LOAD_PLUGINS`` environment variable to a comma-separated list of plugin names to load a controlled subset of plugins instead. + +For example, to load just the ``datasette-vega`` and ``datasette-cluster-map`` plugins, set ``DATASETTE_LOAD_PLUGINS`` to ``datasette-vega,datasette-cluster-map``: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' + datasette mydb.db + +Or: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' \ + datasette mydb.db + +To disable the loading of all additional plugins, set ``DATASETTE_LOAD_PLUGINS`` to an empty string: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='' + datasette mydb.db + +A quick way to test this setting is to use it with the ``datasette plugins`` command: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega' datasette plugins + +This should output the following: + +.. code-block:: json + + [ + { + "name": "datasette-vega", + "static": true, + "templates": false, + "version": "0.6.2", + "hooks": [ + "extra_css_urls", + "extra_js_urls" + ] + } + ] + .. _plugins_installed: Seeing what plugins are installed diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh new file mode 100755 index 00000000..e26d8377 --- /dev/null +++ b/tests/test-datasette-load-plugins.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This should only run in environemnts where both +# datasette-init and datasette-json-html are installed + +PLUGINS=$(datasette plugins) +echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q true || ( \ + echo "Test failed: datasette-json-html not found" && \ + exit 1 \ + ) +# With the DATASETTE_LOAD_PLUGINS we should not see that +PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q false || ( \ + echo "Test failed: datasette-json-html should not have been loaded" && \ + exit 1 \ + ) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ + grep -q true || ( \ + echo "Test failed: datasette-init should have been loaded" && \ + exit 1 \ + ) +# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) +echo "$PLUGINS3"| \ + grep -q '\[\]' || ( \ + echo "Test failed: datasette plugins should have returned []" && \ + exit 1 \ + ) From 2caa53a52a37e53f83e3a854fc721c7e26c5e9ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:24 -0700 Subject: [PATCH 1127/1866] ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 743f5972..d136ad5a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1147,7 +1147,7 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr Datasette's internal database ============================= -Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. From 4c3ef033110407f3b3dbce501659d523724985e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:59 -0700 Subject: [PATCH 1128/1866] Another ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index d136ad5a..540e7058 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1157,7 +1157,7 @@ Plugins can access this database by calling ``internal_db = datasette.get_intern Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``. 2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. 3. Use temporary tables or shared in-memory attached databases when possible. 4. Avoid implementing features that could expose private data stored in the internal database by other plugins. From 9cead33fb9c8704996181f1ab67c7376dee97f15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 10:46:07 -0700 Subject: [PATCH 1129/1866] OperationalError: database table is locked fix See also: - https://til.simonwillison.net/datasette/remember-to-commit --- docs/internals.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 540e7058..6b7d3df8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1012,6 +1012,7 @@ For example: def delete_and_return_count(conn): conn.execute("delete from some_table where id > 5") + conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1028,6 +1029,8 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. +If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. + If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. .. _database_close: From 98ffad9aed15a300e61fb712fa12f177844739b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 15:46:18 -0700 Subject: [PATCH 1130/1866] execute-sql now implies can view instance/database, closes #2169 --- datasette/default_permissions.py | 1 + tests/test_permissions.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 960429fc..5a99d0d8 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -59,6 +59,7 @@ def register_permissions(): takes_database=True, takes_resource=False, default=True, + implies_can_view=True, ), Permission( name="permissions-debug", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index cad0525f..b3987cff 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1183,6 +1183,10 @@ async def test_actor_restrictions( ({"a": ["update-row"]}, "view-instance", None, False), # view-table on a resource implies view-instance ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # execute-sql on a database implies view-instance, view-database + ({"d": {"db1": ["es"]}}, "view-instance", None, True), + ({"d": {"db1": ["es"]}}, "view-database", "db1", True), + ({"d": {"db1": ["es"]}}, "view-database", "db2", False), # update-row on a resource does not imply view-instance ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), # view-database on a resource implies view-instance From fd083e37ec53e7e625111168d324a572344a3b19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 16:06:30 -0700 Subject: [PATCH 1131/1866] Docs for plugins that define more plugin hooks, closes #1765 --- docs/writing_plugins.rst | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a84789b5..a4c96011 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -325,3 +325,60 @@ This object is exposed in templates as the ``urls`` variable, which can be used Back to the Homepage See :ref:`internals_datasette_urls` for full details on this object. + +.. _writing_plugins_extra_hooks: + +Plugins that define new plugin hooks +------------------------------------ + +Plugins can define new plugin hooks that other plugins can use to further extend their functionality. + +`datasette-graphql `__ is one example of a plugin that does this. It defines a new hook called ``graphql_extra_fields``, `described here `__, which other plugins can use to define additional fields that should be included in the GraphQL schema. + +To define additional hooks, add a file to the plugin called ``datasette_your_plugin/hookspecs.py`` with content that looks like this: + +.. code-block:: python + + from pluggy import HookspecMarker + + hookspec = HookspecMarker("datasette") + + @hookspec + def name_of_your_hook_goes_here(datasette): + "Description of your hook." + +You should define your own hook name and arguments here, following the documentation for `Pluggy specifications `__. Make sure to pick a name that is unlikely to clash with hooks provided by any other plugins. + +Then, to register your plugin hooks, add the following code to your ``datasette_your_plugin/__init__.py`` file: + +.. code-block:: python + + from datasette.plugins import pm + from . import hookspecs + + pm.add_hookspecs(hookspecs) + +This will register your plugin hooks as part of the ``datasette`` plugin hook namespace. + +Within your plugin code you can trigger the hook using this pattern: + +.. code-block:: python + + from datasette.plugins import pm + + for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + datasette=datasette + ): + # Do something with plugin_return_value + +Other plugins will then be able to register their own implementations of your hook using this syntax: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def name_of_your_hook_goes_here(datasette): + return "Response from this plugin hook" + +These plugin implementations can accept 0 or more of the named arguments that you defined in your hook specification. From 31d5c4ec05e27165283f0f0004c32227d8b78df8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:43:01 -0700 Subject: [PATCH 1132/1866] Contraction - Google and Microsoft styleguides like it I was trying out https://github.com/errata-ai/vale --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 814d2e67..1a444d0c 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -4,7 +4,7 @@ Authentication and permissions ================================ -Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. +Datasette doesn't require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys. From 05707aa16b5c6c39fbe48b3176b85a8ffe493938 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:50:09 -0700 Subject: [PATCH 1133/1866] click-default-group>=1.2.3 (#2173) * click-default-group>=1.2.3 Now available as a wheel: - https://github.com/click-contrib/click-default-group/issues/21 * Fix for blacken-docs --- docs/writing_plugins.rst | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a4c96011..d0dd8f36 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -343,6 +343,7 @@ To define additional hooks, add a file to the plugin called ``datasette_your_plu hookspec = HookspecMarker("datasette") + @hookspec def name_of_your_hook_goes_here(datasette): "Description of your hook." @@ -366,10 +367,13 @@ Within your plugin code you can trigger the hook using this pattern: from datasette.plugins import pm - for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + for ( + plugin_return_value + ) in pm.hook.name_of_your_hook_goes_here( datasette=datasette ): # Do something with plugin_return_value + pass Other plugins will then be able to register their own implementations of your hook using this syntax: @@ -377,6 +381,7 @@ Other plugins will then be able to register their own implementations of your ho from datasette import hookimpl + @hookimpl def name_of_your_hook_goes_here(datasette): return "Response from this plugin hook" diff --git a/setup.py b/setup.py index 35c9b68b..7d8c1ebc 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( install_requires=[ "asgiref>=3.2.10", "click>=7.1.1", - "click-default-group-wheel>=1.2.2", + "click-default-group>=1.2.3", "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", From e86eaaa4f371512689e973c18879298dab51f80a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Sep 2023 09:16:27 -0700 Subject: [PATCH 1134/1866] Test against Python 3.12 preview (#2175) https://dev.to/hugovk/help-test-python-312-beta-1508/ --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2784db86..656b0b1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - uses: actions/cache@v3 name: Configure pip caching with: From e4abae3fd7a828625d00c35c316852ffbaa5ef2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:34:31 -0700 Subject: [PATCH 1135/1866] Bump Sphinx (#2166) Bumps the python-packages group with 1 update: [sphinx](https://github.com/sphinx-doc/sphinx). - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.4...v7.2.5) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d8c1ebc..c718086b 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.4", + "Sphinx==7.2.5", "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", From fbcb103c0cb6668018ace539a01a6a1f156e8d6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 07:47:24 -0700 Subject: [PATCH 1136/1866] Added example code to database_actions hook documentation --- docs/plugin_hooks.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f8bb203d..84a045b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1439,7 +1439,32 @@ database_actions(datasette, actor, database, request) This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. -Example: `datasette-graphql `_ +This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def database_actions(datasette, actor, database): + async def inner(): + if not await datasette.permission_allowed( + actor, "edit-schema", resource=database, default=False + ): + return [] + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/-/create".format(database) + ), + "label": "Create a table", + } + ] + + return inner + +Example: `datasette-graphql `_, `datasette-edit-schema `_ .. _plugin_hook_skip_csrf: From 2200abfa17f72b1cb741a36b44dc40a04b8ea001 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:49:50 -0700 Subject: [PATCH 1137/1866] Fix for flaky test_hidden_sqlite_stat1_table, closes #2179 --- tests/test_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index f96f571e..93ca43eb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1017,7 +1017,10 @@ async def test_hidden_sqlite_stat1_table(): await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] - assert tables == [("normal", False), ("sqlite_stat1", True)] + assert tables in ( + [("normal", False), ("sqlite_stat1", True)], + [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)], + ) @pytest.mark.asyncio From dbfad6d2201bc65a0c73e699a10c479c1e199e11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:51:09 -0700 Subject: [PATCH 1138/1866] Foreign key label expanding respects table permissions, closes #2178 --- datasette/app.py | 9 ++++++- datasette/facets.py | 2 +- datasette/views/table.py | 2 +- tests/test_table_html.py | 53 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0227f627..618c0ecc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -935,7 +935,7 @@ class Datasette: log_sql_errors=log_sql_errors, ) - async def expand_foreign_keys(self, database, table, column, values): + async def expand_foreign_keys(self, actor, database, table, column, values): """Returns dict mapping (column, value) -> label""" labeled_fks = {} db = self.databases[database] @@ -949,6 +949,13 @@ class Datasette: ][0] except IndexError: return {} + # Ensure user has permission to view the referenced table + if not await self.permission_allowed( + actor=actor, + action="view-table", + resource=(database, fk["other_table"]), + ): + return {} label_column = await db.label_column_for_table(fk["other_table"]) if not label_column: return {(fk["column"], value): str(value) for value in values} diff --git a/datasette/facets.py b/datasette/facets.py index 7fb0c68b..b23615fe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -253,7 +253,7 @@ class ColumnFacet(Facet): # Attempt to expand foreign keys into labels values = [row["value"] for row in facet_rows] expanded = await self.ds.expand_foreign_keys( - self.database, self.table, column, values + self.request.actor, self.database, self.table, column, values ) else: expanded = {} diff --git a/datasette/views/table.py b/datasette/views/table.py index 6df8b915..50ba2b78 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1144,7 +1144,7 @@ async def table_view_data( # Expand them expanded_labels.update( await datasette.expand_foreign_keys( - database_name, table_name, column, values + request.actor, database_name, table_name, column, values ) ) if expanded_labels: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index c4c7878c..e66eb6f0 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1204,3 +1204,56 @@ async def test_format_of_binary_links(size, title, length_bytes): sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text + + +@pytest.mark.asyncio +async def test_foreign_key_labels_obey_permissions(): + ds = Datasette( + metadata={ + "databases": { + "foreign_key_labels": { + "tables": { + # Table a is only visible to root + "a": {"allow": {"id": "root"}}, + } + } + } + } + ) + db = ds.add_memory_database("foreign_key_labels") + await db.execute_write("create table a(id integer primary key, name text)") + await db.execute_write("insert into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") + # Anonymous user can see table b but not table a + blah = await ds.client.get("/foreign_key_labels.json") + anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") + assert anon_a.status_code == 403 + anon_b = await ds.client.get("/foreign_key_labels/b.json?_labels=on") + assert anon_b.status_code == 200 + # root user can see both + cookies = {"ds_actor": ds.sign({"a": {"id": "root"}}, "actor")} + root_a = await ds.client.get( + "/foreign_key_labels/a.json?_labels=on", cookies=cookies + ) + assert root_a.status_code == 200 + root_b = await ds.client.get( + "/foreign_key_labels/b.json?_labels=on", cookies=cookies + ) + assert root_b.status_code == 200 + # Labels should have been expanded for root + assert root_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], + "truncated": False, + } + # But not for anon + assert anon_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": 1}], + "truncated": False, + } From ab040470e2b191c0de48b213193da71e48cd66ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:57:27 -0700 Subject: [PATCH 1139/1866] Applied blacken-docs --- docs/plugin_hooks.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 84a045b0..04fb24ce 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1450,13 +1450,18 @@ This example adds a new database action for creating a table, if the user has th def database_actions(datasette, actor, database): async def inner(): if not await datasette.permission_allowed( - actor, "edit-schema", resource=database, default=False + actor, + "edit-schema", + resource=database, + default=False, ): return [] return [ { "href": datasette.urls.path( - "/-/edit-schema/{}/-/create".format(database) + "/-/edit-schema/{}/-/create".format( + database + ) ), "label": "Create a table", } From c26370485a4fd4bf130da051be9163d92c57f24f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 16:28:30 -0700 Subject: [PATCH 1140/1866] Label expand permission check respects cascade, closes #2178 --- datasette/app.py | 22 ++++++++++------- tests/test_table_html.py | 52 +++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 618c0ecc..ea9739f0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -950,13 +950,19 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table - if not await self.permission_allowed( - actor=actor, - action="view-table", - resource=(database, fk["other_table"]), - ): + other_table = fk["other_table"] + other_column = fk["other_column"] + visible, _ = await self.check_visibility( + actor, + permissions=[ + ("view-table", (database, other_table)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: return {} - label_column = await db.label_column_for_table(fk["other_table"]) + label_column = await db.label_column_for_table(other_table) if not label_column: return {(fk["column"], value): str(value) for value in values} labeled_fks = {} @@ -965,9 +971,9 @@ class Datasette: from {other_table} where {other_column} in ({placeholders}) """.format( - other_column=escape_sqlite(fk["other_column"]), + other_column=escape_sqlite(other_column), label_column=escape_sqlite(label_column), - other_table=escape_sqlite(fk["other_table"]), + other_table=escape_sqlite(other_table), placeholders=", ".join(["?"] * len(set(values))), ) try: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index e66eb6f0..6707665d 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1207,9 +1207,11 @@ async def test_format_of_binary_links(size, title, length_bytes): @pytest.mark.asyncio -async def test_foreign_key_labels_obey_permissions(): - ds = Datasette( - metadata={ +@pytest.mark.parametrize( + "metadata", + ( + # Blocked at table level + { "databases": { "foreign_key_labels": { "tables": { @@ -1218,15 +1220,47 @@ async def test_foreign_key_labels_obey_permissions(): } } } - } - ) + }, + # Blocked at database level + { + "databases": { + "foreign_key_labels": { + # Only root can view this database + "allow": {"id": "root"}, + "tables": { + # But table b is visible to everyone + "b": {"allow": True}, + }, + } + } + }, + # Blocked at the instance level + { + "allow": {"id": "root"}, + "databases": { + "foreign_key_labels": { + "tables": { + # Table b is visible to everyone + "b": {"allow": True}, + } + } + }, + }, + ), +) +async def test_foreign_key_labels_obey_permissions(metadata): + ds = Datasette(metadata=metadata) db = ds.add_memory_database("foreign_key_labels") - await db.execute_write("create table a(id integer primary key, name text)") - await db.execute_write("insert into a (id, name) values (1, 'hello')") await db.execute_write( - "create table b(id integer primary key, name text, a_id integer references a(id))" + "create table if not exists a(id integer primary key, name text)" + ) + await db.execute_write("insert or replace into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table if not exists b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write( + "insert or replace into b (id, name, a_id) values (1, 'world', 1)" ) - await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") # Anonymous user can see table b but not table a blah = await ds.client.get("/foreign_key_labels.json") anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") From b645174271aa08e8ca83b27ff83ce078ecd15da2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:23:59 -0700 Subject: [PATCH 1141/1866] actors_from_ids plugin hook and datasette.actors_from_ids() method (#2181) * Prototype of actors_from_ids plugin hook, refs #2180 * datasette-remote-actors example plugin, refs #2180 --- datasette/app.py | 10 +++++++ datasette/hookspecs.py | 5 ++++ docs/internals.rst | 21 ++++++++++++++ docs/plugin_hooks.rst | 57 ++++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 62 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index ea9739f0..fdec2c86 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -819,6 +819,16 @@ class Datasette: ) return crumbs + async def actors_from_ids( + self, actor_ids: Iterable[Union[str, int]] + ) -> Dict[Union[id, str], Dict]: + result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) + if result is None: + # Do the default thing + return {actor_id: {"id": actor_id} for actor_id in actor_ids} + result = await await_me_maybe(result) + return result + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 801073fc..9069927b 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -94,6 +94,11 @@ def actor_from_request(datasette, request): """Return an actor dictionary based on the incoming request""" +@hookspec(firstresult=True) +def actors_from_ids(datasette, actor_ids): + """Returns a dictionary mapping those IDs to actor dictionaries""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/docs/internals.rst b/docs/internals.rst index 6b7d3df8..13f1d4a1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -322,6 +322,27 @@ await .render_template(template, context=None, request=None) Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +.. _datasette_actors_from_ids: + +await .actors_from_ids(actor_ids) +--------------------------------- + +``actor_ids`` - list of strings or integers + A list of actor IDs to look up. + +Returns a dictionary, where the keys are the IDs passed to it and the values are the corresponding actor dictionaries. + +This method is mainly designed to be used with plugins. See the :ref:`plugin_hook_actors_from_ids` documentation for details. + +If no plugins that implement that hook are installed, the default return value looks like this: + +.. code-block:: json + + { + "1": {"id": "1"}, + "2": {"id": "2"} + } + .. _datasette_permission_allowed: await .permission_allowed(actor, action, resource=None, default=...) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 04fb24ce..e966919b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1071,6 +1071,63 @@ Instead of returning a dictionary, this function can return an awaitable functio Examples: `datasette-auth-tokens `_, `datasette-auth-passwords `_ +.. _plugin_hook_actors_from_ids: + +actors_from_ids(datasette, actor_ids) +------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor_ids`` - list of strings or integers + The actor IDs to look up. + +The hook must return a dictionary that maps the incoming actor IDs to their full dictionary representation. + +Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. + +Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. + +If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. + +The hook can return a dictionary or an awaitable function that then returns a dictionary. + +This example implementation returns actors from a database table: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def actors_from_ids(datasette, actor_ids): + db = datasette.get_database("actors") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + for row in (await db.execute(sql, actor_ids)).rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + +The returned dictionary from this example looks like this: + +.. code-block:: json + + { + "1": {"id": "1", "name": "Tony"}, + "2": {"id": "2", "name": "Tina"}, + } + +These IDs could be integers or strings, depending on how the actors used by the Datasette instance are configured. + +Example: `datasette-remote-actors `_ + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9761fa53..625ae635 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1215,3 +1215,65 @@ async def test_hook_register_permissions_allows_identical_duplicates(): await ds.invoke_startup() # Check that ds.permissions has only one of each assert len([p for p in ds.permissions.values() if p.abbr == "abbr1"]) == 1 + + +@pytest.mark.asyncio +async def test_hook_actors_from_ids(): + # Without the hook should return default {"id": id} list + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("actors_from_ids") + await db.execute_write( + "create table actors (id text primary key, name text, age int)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('3', 'Cate Blanchett', 52)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('5', 'Rooney Mara', 36)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('7', 'Sarah Paulson', 46)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('9', 'Helena Bonham Carter', 55)" + ) + table_names = await db.table_names() + assert table_names == ["actors"] + actors1 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors1 == { + "3": {"id": "3"}, + "5": {"id": "5"}, + "7": {"id": "7"}, + } + + class ActorsFromIdsPlugin: + __name__ = "ActorsFromIdsPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + db = datasette.get_database("actors_from_ids") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + result = await db.execute(sql, actor_ids) + for row in result.rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + + try: + pm.register(ActorsFromIdsPlugin(), name="ActorsFromIdsPlugin") + actors2 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors2 == { + "3": {"id": "3", "name": "Cate Blanchett"}, + "5": {"id": "5", "name": "Rooney Mara"}, + "7": {"id": "7", "name": "Sarah Paulson"}, + } + finally: + pm.unregister(name="ReturnNothingPlugin") From a4c96d01b27ce7cd06662a024da3547132a7c412 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:44:08 -0700 Subject: [PATCH 1142/1866] Release 1.0a6 Refs #1765, #2164, #2169, #2175, #2178, #2181 --- datasette/version.py | 2 +- docs/changelog.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b99c212b..4b65999d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a5" +__version__ = "1.0a6" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 019d6c68..81554f83 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog ========= +.. _v1_0_a6: + +1.0a6 (2023-09-07) +------------------ + +- New plugin hook: :ref:`plugin_hook_actors_from_ids` and an internal method to accompany it, :ref:`datasette_actors_from_ids`. This mechanism is intended to be used by plugins that may need to display the actor who was responsible for something managed by that plugin: they can now resolve the recorded IDs of actors into the full actor objects. (:issue:`2181`) +- ``DATASETTE_LOAD_PLUGINS`` environment variable for :ref:`controlling which plugins ` are loaded by Datasette. (:issue:`2164`) +- Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`) +- The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`) +- Documentation describing a pattern for building plugins that themselves :ref:`define further hooks ` for other plugins. (:issue:`1765`) +- Datasette is now tested against the Python 3.12 preview. (`#2175 `__) + .. _v1_0_a5: 1.0a5 (2023-08-29) From b2ec8717c3619260a1b535eea20e618bf95aa30b Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 13 Sep 2023 14:06:25 -0700 Subject: [PATCH 1143/1866] Plugin configuration now lives in datasette.yaml/json * Checkpoint, moving top-level plugin config to datasette.json * Support database-level and table-level plugin configuration in datasette.yaml Refs #2093 --- datasette/app.py | 48 +++++++++++++++----- docs/configuration.rst | 97 ++++++++++++++++++++++++++++++++++++++-- docs/index.rst | 1 + docs/internals.rst | 2 +- docs/plugin_hooks.rst | 2 +- docs/writing_plugins.rst | 7 +-- tests/conftest.py | 3 +- tests/fixtures.py | 50 ++++++++++++++------- tests/test_cli.py | 38 ++++++++++++++++ tests/test_plugins.py | 23 +++------- 10 files changed, 217 insertions(+), 54 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index fdec2c86..53486007 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -368,7 +368,7 @@ class Datasette: for key in config_settings: if key not in DEFAULT_SETTINGS: raise StartupError("Invalid setting '{}' in datasette.json".format(key)) - + self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions @@ -674,15 +674,43 @@ class Datasette: def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" - plugins = self.metadata( - "plugins", database=database, table=table, fallback=fallback - ) - if plugins is None: - return None - plugin_config = plugins.get(plugin_name) - # Resolve any $file and $env keys - plugin_config = resolve_env_secrets(plugin_config, os.environ) - return plugin_config + if database is None and table is None: + config = self._plugin_config_top(plugin_name) + else: + config = self._plugin_config_nested(plugin_name, database, table, fallback) + + return resolve_env_secrets(config, os.environ) + + def _plugin_config_top(self, plugin_name): + """Returns any top-level plugin configuration for the specified plugin.""" + return ((self.config or {}).get("plugins") or {}).get(plugin_name) + + def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True): + """Returns any database or table-level plugin configuration for the specified plugin.""" + db_config = ((self.config or {}).get("databases") or {}).get(database) + + # if there's no db-level configuration, then return early, falling back to top-level if needed + if not db_config: + return self._plugin_config_top(plugin_name) if fallback else None + + db_plugin_config = (db_config.get("plugins") or {}).get(plugin_name) + + if table: + table_plugin_config = ( + ((db_config.get("tables") or {}).get(table) or {}).get("plugins") or {} + ).get(plugin_name) + + # fallback to db_config or top-level config, in that order, if needed + if table_plugin_config is None and fallback: + return db_plugin_config or self._plugin_config_top(plugin_name) + + return table_plugin_config + + # fallback to top-level if needed + if db_plugin_config is None and fallback: + self._plugin_config_top(plugin_name) + + return db_plugin_config def app_css_hash(self): if not hasattr(self, "_app_css_hash"): diff --git a/docs/configuration.rst b/docs/configuration.rst index ed9975ac..214e9044 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,10 +1,101 @@ .. _configuration: Configuration -======== +============= -Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. +Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: +To facilitate this, You can provide a ``datasette.yaml`` configuration file to datasette with the ``--config``/ ``-c`` flag: + +.. code-block:: bash datasette mydatabase.db --config datasette.yaml + +.. _configuration_reference: + +``datasette.yaml`` reference +---------------------------- + +Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. + +.. tab:: YAML + + .. code-block:: yaml + + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + +.. _configuration_reference_settings: +Settings configuration +~~~~~~~~~~~~~~~~~~~~~~ + +:ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + + +.. _configuration_reference_plugins: +Plugin configuration +~~~~~~~~~~~~~~~~~~~~ + +Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + +For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value diff --git a/docs/index.rst b/docs/index.rst index f5c1f232..cfa3443c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ Contents getting_started installation + configuration ecosystem cli-reference pages diff --git a/docs/internals.rst b/docs/internals.rst index 13f1d4a1..7fc7948c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -296,7 +296,7 @@ The dictionary keys are the permission names - e.g. ``view-instance`` - and the ``table`` - None or string The table the user is interacting with. -This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. +This method lets you read plugin configuration values that were set in ``datasette.yaml``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. The return value will be the value from the configuration file - usually a dictionary. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e966919b..1816d48c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -909,7 +909,7 @@ Potential use-cases: * Run some initialization code for the plugin * Create database tables that a plugin needs on startup -* Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid +* Validate the configuration for a plugin on startup, and raise an error if it is invalid .. note:: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index d0dd8f36..c028b4ff 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -184,7 +184,7 @@ This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` i If there is no configuration for that plugin, the method will return ``None``. -If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so: +If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option inside ``datasette.yaml`` like so: .. [[[cog from metadata_doc import metadata_example @@ -234,11 +234,10 @@ If it cannot find the requested configuration at the table layer, it will fall b In this case, the above code would return that configuration for ANY table within the ``sf-trees`` database. -The plugin configuration could also be set at the top level of ``metadata.yaml``: +The plugin configuration could also be set at the top level of ``datasette.yaml``: .. [[[cog metadata_example(cog, { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", @@ -252,7 +251,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: yaml - title: This is the top-level title in metadata.json plugins: datasette-cluster-map: latitude_column: xlat @@ -264,7 +262,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: json { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", diff --git a/tests/conftest.py b/tests/conftest.py index fb7f768e..31336aea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): @pytest_asyncio.fixture async def ds_client(): from datasette.app import Datasette - from .fixtures import METADATA, PLUGINS_DIR + from .fixtures import CONFIG, METADATA, PLUGINS_DIR global _ds_client if _ds_client is not None: @@ -49,6 +49,7 @@ async def ds_client(): ds = Datasette( metadata=METADATA, + config=CONFIG, plugins_dir=PLUGINS_DIR, settings={ "default_page_size": 50, diff --git a/tests/fixtures.py b/tests/fixtures.py index a6700239..9cf6b605 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -114,6 +114,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, + config=None, metadata=None, crossdb=False, ): @@ -158,6 +159,7 @@ def make_app_client( memory=memory, cors=cors, metadata=metadata or METADATA, + config=config or CONFIG, plugins_dir=PLUGINS_DIR, settings=settings, inspect_data=inspect_data, @@ -296,6 +298,33 @@ def generate_sortable_rows(num): } +CONFIG = { + "plugins": { + "name-of-plugin": {"depth": "root"}, + "env-plugin": {"foo": {"$env": "FOO_ENV"}}, + "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], + "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, + }, + "databases": { + "fixtures": { + "plugins": {"name-of-plugin": {"depth": "database"}}, + "tables": { + "simple_primary_key": { + "plugins": { + "name-of-plugin": { + "depth": "table", + "special": "this-is-simple_primary_key", + } + }, + }, + "sortable": { + "plugins": {"name-of-plugin": {"depth": "table"}}, + }, + }, + } + }, +} + METADATA = { "title": "Datasette Fixtures", "description_html": 'An example SQLite database demonstrating Datasette. Sign in as root user', @@ -306,26 +335,13 @@ METADATA = { "about": "About Datasette", "about_url": "https://github.com/simonw/datasette", "extra_css_urls": ["/static/extra-css-urls.css"], - "plugins": { - "name-of-plugin": {"depth": "root"}, - "env-plugin": {"foo": {"$env": "FOO_ENV"}}, - "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], - "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, - }, "databases": { "fixtures": { "description": "Test tables description", - "plugins": {"name-of-plugin": {"depth": "database"}}, "tables": { "simple_primary_key": { "description_html": "Simple primary key", "title": "This HTML is escaped", - "plugins": { - "name-of-plugin": { - "depth": "table", - "special": "this-is-simple_primary_key", - } - }, }, "sortable": { "sortable_columns": [ @@ -334,7 +350,6 @@ METADATA = { "sortable_with_nulls_2", "text", ], - "plugins": {"name-of-plugin": {"depth": "table"}}, }, "no_primary_key": {"sortable_columns": [], "hidden": True}, "units": {"units": {"distance": "m", "frequency": "Hz"}}, @@ -768,6 +783,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), ) @click.argument("metadata", required=False) +@click.argument("config", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) @@ -782,7 +798,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), help="Write out second test DB to this file", ) -def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): +def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename): """Write out the fixtures database used by Datasette's test suite""" if metadata and not metadata.endswith(".json"): raise click.ClickException("Metadata should end with .json") @@ -805,6 +821,10 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): with open(metadata, "w") as fp: fp.write(json.dumps(METADATA, indent=4)) print(f"- metadata written to {metadata}") + if config: + with open(config, "w") as fp: + fp.write(json.dumps(CONFIG, indent=4)) + print(f"- config written to {config}") if plugins_path: path = pathlib.Path(plugins_path) if not path.exists(): diff --git a/tests/test_cli.py b/tests/test_cli.py index e85bcef1..213db416 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -238,6 +238,44 @@ def test_setting(args): assert settings["default_page_size"] == 5 +def test_plugin_s_overwrite(): + runner = CliRunner() + plugins_dir = str(pathlib.Path(__file__).parent / "plugins") + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=None' + ) + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + "-s", + "plugins.name-of-plugin", + "OVERRIDE", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=OVERRIDE' + ) + + def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 625ae635..37530991 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -234,9 +234,6 @@ async def test_plugin_config(ds_client): async def test_plugin_config_env(ds_client): os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$env": "FOO_ENV"}} == metadata.json()["plugins"]["env-plugin"] del os.environ["FOO_ENV"] @@ -246,11 +243,6 @@ async def test_plugin_config_env_from_list(ds_client): assert [{"in_a_list": "FROM_ENVIRONMENT"}] == ds_client.ds.plugin_config( "env-plugin-list" ) - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert [{"in_a_list": {"$env": "FOO_ENV"}}] == metadata.json()["plugins"][ - "env-plugin-list" - ] del os.environ["FOO_ENV"] @@ -259,11 +251,6 @@ async def test_plugin_config_file(ds_client): with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}} == metadata.json()["plugins"][ - "file-plugin" - ] os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -722,7 +709,7 @@ async def test_hook_register_routes(ds_client, path, body): @pytest.mark.parametrize("configured_path", ("path1", "path2")) def test_hook_register_routes_with_datasette(configured_path): with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": configured_path, @@ -741,7 +728,7 @@ def test_hook_register_routes_with_datasette(configured_path): def test_hook_register_routes_override(): "Plugins can over-ride default paths such as /db/table" with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": "blah", @@ -1099,7 +1086,7 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1151,7 +1138,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): if duplicate == "abbr": abbr2 = "abbr1" ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1186,7 +1173,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): @pytest.mark.asyncio async def test_hook_register_permissions_allows_identical_duplicates(): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ From 16f0b6d8222d06682a31b904d0a402c391ae1c1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Sep 2023 14:15:32 -0700 Subject: [PATCH 1144/1866] JSON/YAML tabs on configuration docs page --- docs/configuration.rst | 171 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 214e9044..4a7258b9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -18,6 +18,40 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + """) + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -48,12 +82,61 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_page_size": 50, + "sql_time_limit_ms": 3500, + "max_returned_rows": 2000 + }, + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "databases": { + "your_db_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "tables": { + "your_table_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueB" + } + } + } + } + } + } + } +.. [[[end]]] + .. _configuration_reference_settings: Settings configuration ~~~~~~~~~~~~~~~~~~~~~~ :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -63,6 +146,17 @@ Settings configuration default_allow_sql: off default_page_size: 50 +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_allow_sql": "off", + "default_page_size": 50 + } + } +.. [[[end]]] .. _configuration_reference_plugins: Plugin configuration @@ -70,6 +164,19 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -79,8 +186,44 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } +.. [[[end]]] + For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -99,3 +242,31 @@ For database level or table level plugin configuration, nest it under the approp plugins: datasette-my-plugin: key: my_value + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "my_database": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + }, + "my_other_database": { + "tables": { + "my_table": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } + } + } + } + } +.. [[[end]]] \ No newline at end of file From 852f5014853943fa27f43ddaa2d442545b3259fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 Sep 2023 09:35:18 -0700 Subject: [PATCH 1145/1866] Switch from pkg_resources to importlib.metadata in app.py, refs #2057 --- datasette/app.py | 6 +++--- tests/test_plugins.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 53486007..c0e80700 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -8,11 +8,11 @@ import functools import glob import hashlib import httpx +import importlib.metadata import inspect from itsdangerous import BadSignature import json import os -import pkg_resources import re import secrets import sys @@ -1118,9 +1118,9 @@ class Datasette: if using_pysqlite3: for package in ("pysqlite3", "pysqlite3-binary"): try: - info["pysqlite3"] = pkg_resources.get_distribution(package).version + info["pysqlite3"] = importlib.metadata.version(package) break - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: pass return info diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 37530991..3bc117f3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1264,3 +1264,25 @@ async def test_hook_actors_from_ids(): } finally: pm.unregister(name="ReturnNothingPlugin") + + +@pytest.mark.asyncio +async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + + class DummyPlugin: + __name__ = "DummyPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + return {} + + try: + pm.register(DummyPlugin(), name="DummyPlugin") + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + installed_plugins = {p["name"] for p in response.json()} + assert "DummyPlugin" in installed_plugins + + finally: + pm.unregister(name="DummyPlugin") From f56e043747bde4faa1d78588636df6c0dadebc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:39:11 -0700 Subject: [PATCH 1146/1866] test_facet_against_in_memory_database, refs #2189 This is meant to illustrate a crashing bug but it does not trigger it. --- tests/test_facets.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_facets.py b/tests/test_facets.py index 48cc0ff2..a68347f0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -643,3 +643,50 @@ async def test_conflicting_facet_names_json(ds_client): "created_2", "tags_2", } + + +@pytest.mark.asyncio +async def test_facet_against_in_memory_database(): + ds = Datasette() + db = ds.add_memory_database("mem") + await db.execute_write("create table t (id integer primary key, name text)") + to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] + await db.execute_write_many("insert into t (name) values (?)", to_insert) + response1 = await ds.client.get("/mem/t.json") + assert response1.status_code == 200 + response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + assert response2.status_code == 200 + assert response2.json() == { + "ok": True, + "next": None, + "facet_results": { + "results": { + "name": { + "name": "name", + "type": "column", + "hideable": True, + "toggle_url": "/mem/t.json?_size=0", + "results": [ + { + "value": "one", + "label": "one", + "count": 800, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", + "selected": False, + }, + { + "value": "two", + "label": "two", + "count": 300, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", + "selected": False, + }, + ], + "truncated": False, + } + }, + "timed_out": [], + }, + "rows": [], + "truncated": False, + } From 6ed7908580fa2ba9297c3225d85c56f8b08b9937 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:44:13 -0700 Subject: [PATCH 1147/1866] Simplified test for #2189 This now executes two facets, in the hope that parallel facet execution would illustrate the bug - but it did not illustrate the bug. --- tests/test_facets.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/tests/test_facets.py b/tests/test_facets.py index a68347f0..85c8f85b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -649,44 +649,17 @@ async def test_conflicting_facet_names_json(ds_client): async def test_facet_against_in_memory_database(): ds = Datasette() db = ds.add_memory_database("mem") - await db.execute_write("create table t (id integer primary key, name text)") - to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] - await db.execute_write_many("insert into t (name) values (?)", to_insert) - response1 = await ds.client.get("/mem/t.json") + await db.execute_write( + "create table t (id integer primary key, name text, name2 text)" + ) + to_insert = [{"name": "one", "name2": "1"} for _ in range(800)] + [ + {"name": "two", "name2": "2"} for _ in range(300) + ] + print(to_insert) + await db.execute_write_many( + "insert into t (name, name2) values (:name, :name2)", to_insert + ) + response1 = await ds.client.get("/mem/t") assert response1.status_code == 200 - response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + response2 = await ds.client.get("/mem/t?_facet=name&_facet=name2") assert response2.status_code == 200 - assert response2.json() == { - "ok": True, - "next": None, - "facet_results": { - "results": { - "name": { - "name": "name", - "type": "column", - "hideable": True, - "toggle_url": "/mem/t.json?_size=0", - "results": [ - { - "value": "one", - "label": "one", - "count": 800, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", - "selected": False, - }, - { - "value": "two", - "label": "two", - "count": 300, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", - "selected": False, - }, - ], - "truncated": False, - } - }, - "timed_out": [], - }, - "rows": [], - "truncated": False, - } From b0e5d8afa308759f4ee9f3ecdf61101dffc4a037 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 Sep 2023 15:10:55 -0700 Subject: [PATCH 1148/1866] Stop using parallel SQL queries for tables Refs: - #2189 --- datasette/views/table.py | 16 ++++++---------- docs/internals.rst | 1 + 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50ba2b78..4f4baeed 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -74,11 +74,10 @@ class Row: return json.dumps(d, default=repr, indent=2) -async def _gather_parallel(*args): - return await asyncio.gather(*args) - - -async def _gather_sequential(*args): +async def run_sequential(*args): + # This used to be swappable for asyncio.gather() to run things in + # parallel, but this lead to hard-to-debug locking issues with + # in-memory databases: https://github.com/simonw/datasette/issues/2189 results = [] for fn in args: results.append(await fn) @@ -1183,9 +1182,6 @@ async def table_view_data( ) rows = rows[:page_size] - # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather - gather = _gather_sequential if request.args.get("_noparallel") else _gather_parallel - # Resolve extras extras = _get_extras(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): @@ -1249,7 +1245,7 @@ async def table_view_data( if not nofacet: # Run them in parallel facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await gather(*facet_awaitables) + facet_awaitable_results = await run_sequential(*facet_awaitables) for ( instance_facet_results, instance_facets_timed_out, @@ -1282,7 +1278,7 @@ async def table_view_data( ): # Run them in parallel facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await gather(*facet_suggest_awaitables): + for suggest_result in await run_sequential(*facet_suggest_awaitables): suggested_facets.extend(suggest_result) return suggested_facets diff --git a/docs/internals.rst b/docs/internals.rst index 7fc7948c..4e9a6747 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1317,6 +1317,7 @@ This example uses the :ref:`register_routes() ` plugin h (r"/parallel-queries$", parallel_queries), ] +Note that running parallel SQL queries in this way has `been known to cause problems in the past `__, so treat this example with caution. Adding ``?_trace=1`` will show that the trace covers both of those child tasks. From 6763572948ffd047a89a3bbf7c300e91f51ae98f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:11:24 -0700 Subject: [PATCH 1149/1866] Bump sphinx, furo, black Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [black](https://github.com/psf/black). Updates `sphinx` from 7.2.5 to 7.2.6 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.5...v7.2.6) Updates `furo` from 2023.8.19 to 2023.9.10 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.08.19...2023.09.10) Updates `black` from 23.7.0 to 23.9.1 - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.7.0...23.9.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c718086b..415fd27c 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.5", - "furo==2023.8.19", + "Sphinx==7.2.6", + "furo==2023.9.10", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -83,7 +83,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.7.0", + "black==23.9.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From 10bc80547330e826a749ce710da21ae29f7e6048 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:11:35 -0700 Subject: [PATCH 1150/1866] Finish removing pkg_resources, closes #2057 --- datasette/plugins.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 6ec08a81..017f3b9d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,7 +1,7 @@ -import importlib +import importlib.metadata +import importlib.resources import os import pluggy -import pkg_resources import sys from . import hookspecs @@ -35,15 +35,15 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = pkg_resources.get_distribution(package_name) - entry_map = distribution.get_entry_map() - if "datasette" in entry_map: - for plugin_name, entry_point in entry_map["datasette"].items(): + distribution = importlib.metadata.distribution(package_name) + entry_points = distribution.entry_points + for entry_point in entry_points: + if entry_point.group == "datasette": mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -61,16 +61,16 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if pkg_resources.resource_isdir(plugin.__name__, "static"): - static_path = pkg_resources.resource_filename( - plugin.__name__, "static" + if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + static_path = str( + importlib.resources.files(plugin.__name__) / "static" ) - if pkg_resources.resource_isdir(plugin.__name__, "templates"): - templates_path = pkg_resources.resource_filename( - plugin.__name__, "templates" + if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + templates_path = str( + importlib.resources.files(plugin.__name__) / "templates" ) - except (KeyError, ImportError): - # Caused by --plugins_dir= plugins - KeyError/ImportError thrown in Py3.5 + except (TypeError, ModuleNotFoundError): + # Caused by --plugins_dir= plugins pass plugin_info = { "name": plugin.__name__, @@ -81,6 +81,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.project_name + plugin_info["name"] = distinfo.name plugins.append(plugin_info) return plugins From 947520c1fe940de79f5db856dd693330f1bbf547 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:31:32 -0700 Subject: [PATCH 1151/1866] Release notes for 0.64.4 on main --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81554f83..52e1db3b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_4: + +0.64.4 (2023-09-21) +------------------- + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v1_0_a6: 1.0a6 (2023-09-07) From b0d0a0e5de8bb5b9b6c253e8af451a532266bcf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:42:15 -0700 Subject: [PATCH 1152/1866] importlib_resources for Python < 3.9, refs #2057 --- datasette/plugins.py | 15 ++++++++++----- setup.py | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 017f3b9d..a93145cf 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,10 +1,15 @@ import importlib.metadata -import importlib.resources import os import pluggy import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -61,13 +66,13 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + if (importlib_resources.files(plugin.__name__) / "static").is_dir(): static_path = str( - importlib.resources.files(plugin.__name__) / "static" + importlib_resources.files(plugin.__name__) / "static" ) - if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + if (importlib_resources.files(plugin.__name__) / "templates").is_dir(): templates_path = str( - importlib.resources.files(plugin.__name__) / "templates" + importlib_resources.files(plugin.__name__) / "templates" ) except (TypeError, ModuleNotFoundError): # Caused by --plugins_dir= plugins diff --git a/setup.py b/setup.py index 415fd27c..a2728f6b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", + 'importlib_resources>=1.3.1; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 80a9cd9620fddf2695d12d8386a91e7c6b145ef2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:55:50 -0700 Subject: [PATCH 1153/1866] test-datasette-load-plugins now fails correctly, refs #2193 --- tests/test-datasette-load-plugins.sh | 41 +++++++++++++--------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh index e26d8377..03e08bb1 100755 --- a/tests/test-datasette-load-plugins.sh +++ b/tests/test-datasette-load-plugins.sh @@ -3,27 +3,24 @@ # datasette-init and datasette-json-html are installed PLUGINS=$(datasette plugins) -echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q true || ( \ - echo "Test failed: datasette-json-html not found" && \ - exit 1 \ - ) -# With the DATASETTE_LOAD_PLUGINS we should not see that +if ! echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q true; then + echo "Test failed: datasette-json-html not found" + exit 1 +fi + PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q false || ( \ - echo "Test failed: datasette-json-html should not have been loaded" && \ - exit 1 \ - ) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ - grep -q true || ( \ - echo "Test failed: datasette-init should have been loaded" && \ - exit 1 \ - ) -# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q false; then + echo "Test failed: datasette-json-html should not have been loaded" + exit 1 +fi + +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | grep -q true; then + echo "Test failed: datasette-init should have been loaded" + exit 1 +fi + PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) -echo "$PLUGINS3"| \ - grep -q '\[\]' || ( \ - echo "Test failed: datasette plugins should have returned []" && \ - exit 1 \ - ) +if ! echo "$PLUGINS3" | grep -q '\[\]'; then + echo "Test failed: datasette plugins should have returned []" + exit 1 +fi From b7cf0200e21796a6ff653c6f94a4ee5fcfde0346 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:22:40 -0700 Subject: [PATCH 1154/1866] Swap order of config and metadata options, refs #2194 --- tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 9cf6b605..16aa234e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -782,8 +782,8 @@ def assert_permissions_checked(datasette, actions): default="fixtures.db", type=click.Path(file_okay=True, dir_okay=False), ) -@click.argument("metadata", required=False) @click.argument("config", required=False) +@click.argument("metadata", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) From 2da1a6acec915b81a16127008fd739c7d6075681 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:26:13 -0700 Subject: [PATCH 1155/1866] Use importlib_metadata for Python 3.8, refs #2057 --- datasette/plugins.py | 10 ++++++---- setup.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index a93145cf..f23f5cfb 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,4 @@ -import importlib.metadata +import importlib import os import pluggy import sys @@ -6,8 +6,10 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources + import importlib.metadata as importlib_metadata else: import importlib_resources + import importlib_metadata DEFAULT_PLUGINS = ( @@ -40,7 +42,7 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = importlib.metadata.distribution(package_name) + distribution = importlib_metadata.distribution(package_name) entry_points = distribution.entry_points for entry_point in entry_points: if entry_point.group == "datasette": @@ -48,7 +50,7 @@ if DATASETTE_LOAD_PLUGINS is not None: pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except importlib.metadata.PackageNotFoundError: + except importlib_metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -86,6 +88,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.name + plugin_info["name"] = distinfo.name or distinfo.project_name plugins.append(plugin_info) return plugins diff --git a/setup.py b/setup.py index a2728f6b..65a3b335 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From f130c7c0a88e50cea4121ea18d1f6db2431b6fab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:09:57 -0700 Subject: [PATCH 1156/1866] Deploy with fixtures-metadata.json, refs #2194, #2195 --- .github/workflows/deploy-latest.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 0dfa5a60..e0405440 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -38,8 +38,14 @@ jobs: run: | pytest -n auto -m "not serial" pytest -m "serial" - - name: Build fixtures.db - run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db + - name: Build fixtures.db and other files needed to deploy the demo + run: |- + python tests/fixtures.py \ + fixtures.db \ + fixtures-config.json \ + fixtures-metadata.json \ + plugins \ + --extra-db-filename extra_database.db - name: Build docs.db if: ${{ github.ref == 'refs/heads/main' }} run: |- @@ -88,13 +94,13 @@ jobs: } return queries EOF - - name: Make some modifications to metadata.json - run: | - cat fixtures.json | \ - jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ - jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ - > metadata.json - cat metadata.json + # - name: Make some modifications to metadata.json + # run: | + # cat fixtures.json | \ + # jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ + # jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ + # > metadata.json + # cat metadata.json - name: Set up Cloud Run uses: google-github-actions/setup-gcloud@v0 with: @@ -112,7 +118,7 @@ jobs: # Replace 1.0 with one-dot-zero in SUFFIX export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m metadata.json \ + -m fixtures-metadata.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ From e4f868801a6633400045f59584cfe650961c3fa6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:58:39 -0700 Subject: [PATCH 1157/1866] Use importlib_metadata for 3.9 as well, refs #2057 --- datasette/plugins.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index f23f5cfb..1ed3747f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -6,9 +6,11 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources - import importlib.metadata as importlib_metadata else: import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: import importlib_metadata diff --git a/setup.py b/setup.py index 65a3b335..d09a9e3d 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', - 'importlib_metadata>=4.6; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.10"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 836b1587f08800658c63679d850f0149003c5311 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 15:06:19 -0700 Subject: [PATCH 1158/1866] Release notes for 1.0a7 Refs #2189 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 4b65999d..55e2cd42 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a6" +__version__ = "1.0a7" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 52e1db3b..9a5290c0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a7: + +1.0a7 (2023-09-21) +------------------ + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v0_64_4: 0.64.4 (2023-09-21) From d51e63d3bb3e32f80d1c0f04adff7c1dd5a7b0c0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:03:37 -0700 Subject: [PATCH 1159/1866] Release notes for 0.64.5, refs #2197 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a5290c0..48bf9ef5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_5: + +0.64.5 (2023-10-08) +------------------- + +- Dropped dependency on ``click-default-group-wheel``, which could cause a dependency conflict. (:issue:`2197`) + .. _v1_0_a7: 1.0a7 (2023-09-21) From 85a41987c7753c3af92ba6b8b6007211eb46602f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:07:11 -0700 Subject: [PATCH 1160/1866] Fixed typo acepts -> accepts --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1816d48c..eb6bf4ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -488,7 +488,7 @@ This will register ``render_demo`` to be called when paths with the extension `` ``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls. -``can_render_demo`` is a Python function (or ``async def`` function) which acepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. +``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. From 4e1188f60f8b4f90c32a372f3f70a26a3ebb88ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:09:45 -0700 Subject: [PATCH 1161/1866] Upgrade spellcheck.yml workflow --- .github/workflows/spellcheck.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 722e5c68..0ce9e10c 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,18 +9,13 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 with: - python-version: 3.11 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install dependencies run: | pip install -e '.[docs]' From 35deaabcb105903790d18710a26e77545f6852ce Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 12 Oct 2023 09:16:37 -0700 Subject: [PATCH 1162/1866] Move non-metadata configuration from metadata.yaml to datasette.yaml * Allow and permission blocks moved to datasette.yaml * Documentation updates, initial framework for configuration reference --- datasette/app.py | 6 +- datasette/default_permissions.py | 37 ++-- docs/authentication.rst | 338 ++++++++++++++---------------- docs/configuration.rst | 64 ++++-- docs/custom_templates.rst | 137 ++++++------ docs/facets.rst | 12 +- docs/full_text_search.rst | 4 +- docs/internals.rst | 2 +- docs/metadata.rst | 126 ++++++++--- docs/metadata_doc.py | 21 +- docs/plugins.rst | 30 +-- docs/settings.rst | 5 +- docs/sql_queries.rst | 56 ++--- docs/writing_plugins.rst | 8 +- tests/fixtures.py | 48 ++--- tests/test_canned_queries.py | 14 +- tests/test_html.py | 44 ++-- tests/test_internals_datasette.py | 6 +- tests/test_permissions.py | 138 ++++++------ tests/test_plugins.py | 4 +- tests/test_table_api.py | 2 +- tests/test_table_html.py | 8 +- 22 files changed, 606 insertions(+), 504 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c0e80700..7dfc63c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -721,7 +721,9 @@ class Datasette: return self._app_css_hash async def get_canned_queries(self, database_name, actor): - queries = self.metadata("queries", database=database_name, fallback=False) or {} + queries = ( + ((self.config or {}).get("databases") or {}).get(database_name) or {} + ).get("queries") or {} for more_queries in pm.hook.canned_queries( datasette=self, database=database_name, @@ -1315,7 +1317,7 @@ class Datasette: ): hook = await await_me_maybe(hook) collected.extend(hook) - collected.extend(self.metadata(key) or []) + collected.extend((self.config or {}).get(key) or []) output = [] for url_or_dict in collected: if isinstance(url_or_dict, dict): diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 5a99d0d8..d29dbe84 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,14 +144,14 @@ def permission_allowed_default(datasette, actor, action, resource): "view-query", "execute-sql", ): - result = await _resolve_metadata_view_permissions( + result = await _resolve_config_view_permissions( datasette, actor, action, resource ) if result is not None: return result # Check custom permissions: blocks - result = await _resolve_metadata_permissions_blocks( + result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) if result is not None: @@ -164,10 +164,10 @@ def permission_allowed_default(datasette, actor, action, resource): return inner -async def _resolve_metadata_permissions_blocks(datasette, actor, action, resource): +async def _resolve_config_permissions_blocks(datasette, actor, action, resource): # Check custom permissions: blocks - metadata = datasette.metadata() - root_block = (metadata.get("permissions", None) or {}).get(action) + config = datasette.config or {} + root_block = (config.get("permissions", None) or {}).get(action) if root_block: root_result = actor_matches_allow(actor, root_block) if root_result is not None: @@ -180,7 +180,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc else: database = resource[0] database_block = ( - (metadata.get("databases", {}).get(database, {}).get("permissions", None)) or {} + (config.get("databases", {}).get(database, {}).get("permissions", None)) or {} ).get(action) if database_block: database_result = actor_matches_allow(actor, database_block) @@ -192,7 +192,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc database, table_or_query = resource table_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("tables", {}) .get(table_or_query, {}) @@ -207,7 +207,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc # Finally the canned queries query_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("queries", {}) .get(table_or_query, {}) @@ -222,25 +222,30 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc return None -async def _resolve_metadata_view_permissions(datasette, actor, action, resource): +async def _resolve_config_view_permissions(datasette, actor, action, resource): + config = datasette.config or {} if action == "view-instance": - allow = datasette.metadata("allow") + allow = config.get("allow") if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - database_allow = datasette.metadata("allow", database=resource) + database_allow = ((config.get("databases") or {}).get(resource) or {}).get( + "allow" + ) if database_allow is None: return None return actor_matches_allow(actor, database_allow) elif action == "view-table": database, table = resource - tables = datasette.metadata("tables", database=database) or {} + tables = ((config.get("databases") or {}).get(database) or {}).get( + "tables" + ) or {} table_allow = (tables.get(table) or {}).get("allow") if table_allow is None: return None return actor_matches_allow(actor, table_allow) elif action == "view-query": - # Check if this query has a "allow" block in metadata + # Check if this query has a "allow" block in config database, query_name = resource query = await datasette.get_canned_query(database, query_name, actor) assert query is not None @@ -250,9 +255,11 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, allow) elif action == "execute-sql": # Use allow_sql block from database block, or from top-level - database_allow_sql = datasette.metadata("allow_sql", database=resource) + database_allow_sql = ((config.get("databases") or {}).get(resource) or {}).get( + "allow_sql" + ) if database_allow_sql is None: - database_allow_sql = datasette.metadata("allow_sql") + database_allow_sql = config.get("allow_sql") if database_allow_sql is None: return None return actor_matches_allow(actor, database_allow_sql) diff --git a/docs/authentication.rst b/docs/authentication.rst index 1a444d0c..a301113a 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -67,7 +67,7 @@ An **action** is a string describing the action the actor would like to perform. A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource. -Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. +Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. @@ -76,7 +76,7 @@ Permissions with potentially harmful effects should default to *deny*. Plugin au Defining permissions with "allow" blocks ---------------------------------------- -The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perform a permission. +The standard way to define permissions in Datasette is to use an ``"allow"`` block :ref:`in the datasette.yaml file `. This is a JSON document describing which actors are allowed to perform a permission. The most basic form of allow block is this (`allow demo `__, `deny demo `__): @@ -186,18 +186,18 @@ The /-/allow-debug tool The ``/-/allow-debug`` tool lets you try out different ``"action"`` blocks against different ``"actor"`` JSON objects. You can try that out here: https://latest.datasette.io/-/allow-debug -.. _authentication_permissions_metadata: +.. _authentication_permissions_config: -Access permissions in metadata -============================== +Access permissions in ``datasette.yaml`` +======================================== -There are two ways to configure permissions using ``metadata.json`` (or ``metadata.yaml``). +There are two ways to configure permissions using ``datasette.yaml`` (or ``datasette.json``). For simple visibility permissions you can use ``"allow"`` blocks in the root, database, table and query sections. For other permissions you can use a ``"permissions"`` block, described :ref:`in the next section `. -You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`metadata` configuration. +You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`configuration`. You can control the following: @@ -216,25 +216,25 @@ Access to an instance Here's how to restrict access to your entire Datasette instance to just the ``"id": "root"`` user: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "title": "My private Datasette instance", - "allow": { - "id": "root" - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ title: My private Datasette instance allow: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + title: My private Datasette instance + allow: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -249,21 +249,22 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i To deny access to all users, you can use ``"allow": false``: .. [[[cog - metadata_example(cog, { - "title": "My entirely inaccessible instance", - "allow": False - }) + config_example(cog, """ + title: My entirely inaccessible instance + allow: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - title: My entirely inaccessible instance - allow: false + + title: My entirely inaccessible instance + allow: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -283,28 +284,26 @@ Access to specific databases To limit access to a specific ``private.db`` database to just authenticated users, use the ``"allow"`` block like this: .. [[[cog - metadata_example(cog, { - "databases": { - "private": { - "allow": { - "id": "*" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: private: allow: - id: '*' + id: "*" + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + private: + allow: + id: "*" + + +.. tab:: datasette.json .. code-block:: json @@ -327,34 +326,30 @@ Access to specific tables and views To limit access to the ``users`` table in your ``bakery.db`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "bakery": { - "tables": { - "users": { - "allow": { - "id": "*" - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: bakery: tables: users: allow: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + bakery: + tables: + users: + allow: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -385,32 +380,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific canned queries --------------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``metadata.json`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog - metadata_example(cog, { - "databases": { - "dogs": { - "queries": { - "add_name": { - "sql": "INSERT INTO names (name) VALUES (:name)", - "write": True, - "allow": { - "id": ["root"] - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: dogs: queries: @@ -420,9 +395,26 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to allow: id: - root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + dogs: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + allow: + id: + - root + + +.. tab:: datasette.json .. code-block:: json @@ -461,19 +453,20 @@ You can alternatively use an ``"allow_sql"`` block to control who is allowed to To prevent any user from executing arbitrary SQL queries, use this: .. [[[cog - metadata_example(cog, { - "allow_sql": False - }) + config_example(cog, """ + allow_sql: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: false + + allow_sql: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -485,22 +478,22 @@ To prevent any user from executing arbitrary SQL queries, use this: To enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following: .. [[[cog - metadata_example(cog, { - "allow_sql": { - "id": "root" - } - }) + config_example(cog, """ + allow_sql: + id: root + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: - id: root + + allow_sql: + id: root -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -514,28 +507,26 @@ To enable just the :ref:`root user` to execute SQL for all To limit this ability for just one specific database, use this: .. [[[cog - metadata_example(cog, { - "databases": { - "mydatabase": { - "allow_sql": { - "id": "root" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: mydatabase: allow_sql: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + mydatabase: + allow_sql: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -552,33 +543,32 @@ To limit this ability for just one specific database, use this: .. _authentication_permissions_other: -Other permissions in metadata -============================= +Other permissions in ``datasette.yaml`` +======================================= -For all other permissions, you can use one or more ``"permissions"`` blocks in your metadata. +For all other permissions, you can use one or more ``"permissions"`` blocks in your ``datasette.yaml`` configuration file. -To grant access to the :ref:`permissions debug tool ` to all signed in users you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your metadata: +To grant access to the :ref:`permissions debug tool ` to all signed in users, you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your configuration: .. [[[cog - metadata_example(cog, { - "permissions": { - "debug-menu": { - "id": "*" - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ permissions: debug-menu: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + permissions: + debug-menu: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -594,31 +584,28 @@ To grant access to the :ref:`permissions debug tool ` to a To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "permissions": { - "create-table": { - "id": "editor" - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: permissions: create-table: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + permissions: + create-table: + id: editor + + +.. tab:: datasette.json .. code-block:: json @@ -638,27 +625,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "tables": { - "reports": { - "permissions": { - "insert-row": { - "id": "editor" - } - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: tables: @@ -666,9 +633,24 @@ And for ``insert-row`` against the ``reports`` table in that ``docs`` database: permissions: insert-row: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + tables: + reports: + permissions: + insert-row: + id: editor + + +.. tab:: datasette.json .. code-block:: json diff --git a/docs/configuration.rst b/docs/configuration.rst index 4a7258b9..4e108602 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,15 +13,15 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d .. _configuration_reference: -``datasette.yaml`` reference +``datasette.yaml`` Reference ---------------------------- Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # Datasette settings block settings: @@ -52,10 +52,11 @@ Here's a full example of all the valid configuration options that can exist insi ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml + # Datasette settings block settings: default_page_size: 50 @@ -82,7 +83,8 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB -.. tab:: JSON + +.. tab:: datasette.json .. code-block:: json @@ -125,9 +127,9 @@ Settings configuration :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml settings: @@ -137,7 +139,7 @@ Settings configuration ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -146,7 +148,7 @@ Settings configuration default_allow_sql: off default_page_size: 50 -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -165,9 +167,9 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml plugins: @@ -177,7 +179,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -186,7 +188,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -202,9 +204,9 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml databases: @@ -224,7 +226,7 @@ For database level or table level plugin configuration, nest it under the approp ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -243,7 +245,7 @@ For database level or table level plugin configuration, nest it under the approp datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -269,4 +271,30 @@ For database level or table level plugin configuration, nest it under the approp } } } -.. [[[end]]] \ No newline at end of file +.. [[[end]]] + + +.. _configuration_reference_permissions: +Permissions Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + + +.. _configuration_reference_authentication: +Authentication Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_canned_queries: +Canned Queries Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_css_js: +Extra CSS and JS Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c0f64cb5..d8e4ac96 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -10,35 +10,34 @@ Datasette provides a number of ways of customizing the way data is displayed. Custom CSS and JavaScript ------------------------- -When you launch Datasette, you can specify a custom metadata file like this:: +When you launch Datasette, you can specify a custom configuration file like this:: - datasette mydb.db --metadata metadata.yaml + datasette mydb.db --config datasette.yaml -Your ``metadata.yaml`` file can include links that look like this: +Your ``datasette.yaml`` file can include links that look like this: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ extra_css_urls: - https://simonwillison.net/static/css/all.bf8cd891642c.css extra_js_urls: - https://code.jquery.com/jquery-3.2.1.slim.min.js + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - https://simonwillison.net/static/css/all.bf8cd891642c.css + extra_js_urls: + - https://code.jquery.com/jquery-3.2.1.slim.min.js + + +.. tab:: datasette.json .. code-block:: json @@ -62,35 +61,30 @@ The extra CSS and JavaScript files will be linked in the ```` of every pag You can also specify a SRI (subresource integrity hash) for these assets: .. [[[cog - metadata_example(cog, { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ extra_css_urls: - url: https://simonwillison.net/static/css/all.bf8cd891642c.css sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI extra_js_urls: - url: https://code.jquery.com/jquery-3.2.1.slim.min.js sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json .. code-block:: json @@ -115,7 +109,7 @@ This will produce: .. code-block:: html + {% for url in extra_js_urls %} {% endfor %} diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py new file mode 100644 index 00000000..7db45464 --- /dev/null +++ b/demos/plugins/example_js_manager_plugins.py @@ -0,0 +1,21 @@ +from datasette import hookimpl + +# Test command: +# datasette fixtures.db \ --plugins-dir=demos/plugins/ +# \ --static static:demos/plugins/static + +# Create a set with view names that qualify for this JS, since plugins won't do anything on other pages +# Same pattern as in Nteract data explorer +# https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/datasette_nteract_data_explorer/__init__.py#L77 +PERMITTED_VIEWS = {"table", "query", "database"} + + +@hookimpl +def extra_js_urls(view_name): + print(view_name) + if view_name in PERMITTED_VIEWS: + return [ + { + "url": f"/static/table-example-plugins.js", + } + ] diff --git a/demos/plugins/static/table-example-plugins.js b/demos/plugins/static/table-example-plugins.js new file mode 100644 index 00000000..8c19d9a6 --- /dev/null +++ b/demos/plugins/static/table-example-plugins.js @@ -0,0 +1,100 @@ +/** + * Example usage of Datasette JS Manager API + */ + +document.addEventListener("datasette_init", function (evt) { + const { detail: manager } = evt; + // === Demo plugins: remove before merge=== + addPlugins(manager); +}); + +/** + * Examples for to test datasette JS api + */ +const addPlugins = (manager) => { + + manager.registerPlugin("column-name-plugin", { + version: 0.1, + makeColumnActions: (columnMeta) => { + const { column } = columnMeta; + + return [ + { + label: "Copy name to clipboard", + onClick: (evt) => copyToClipboard(column), + }, + { + label: "Log column metadata to console", + onClick: (evt) => console.log(column), + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-graphs", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: "First", + render: node => { + const description = document.createElement('p'); + description.innerText = 'Hello world'; + node.appendChild(description); + } + }, + { + id: 'second-panel', + label: "Second", + render: node => { + const iframe = document.createElement('iframe'); + iframe.src = "https://observablehq.com/embed/@d3/sortable-bar-chart?cell=viewof+order&cell=chart"; + iframe.width = 800; + iframe.height = 635; + iframe.frameborder = '0'; + node.appendChild(iframe); + } + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-maps", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + // ID only has to be unique within a plugin, manager namespaces for you + id: 'first-map-panel', + label: "Map plugin", + // datasette-vega, leafleft can provide a "render" function + render: node => node.innerHTML = "Here sits a map", + }, + { + id: 'second-panel', + label: "Image plugin", + render: node => { + const img = document.createElement('img'); + img.src = 'https://datasette.io/static/datasette-logo.svg' + node.appendChild(img); + }, + } + ]; + }, + }); + + // Future: dispatch message to some other part of the page with CustomEvent API + // Could use to drive filter/sort query builder actions without page refresh. +} + + + +async function copyToClipboard(str) { + try { + await navigator.clipboard.writeText(str); + } catch (err) { + /** Rejected - text failed to copy to the clipboard. Browsers didn't give permission */ + console.error('Failed to copy: ', err); + } +} From 067cc75dfa01612f9a47815b33804361e18bf5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Dec 2023 09:49:04 -0800 Subject: [PATCH 1168/1866] Fixed broken example links in row page documentation --- docs/pages.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages.rst b/docs/pages.rst index 0ae72351..2ce05428 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -70,10 +70,10 @@ Table cells with extremely long text contents are truncated on the table view ac Rows which are the targets of foreign key references from other tables will show a link to a filtered search for all records that reference that row. Here's an example from the Registers of Members Interests database: -`../people/uk.org.publicwhip%2Fperson%2F10001 `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001 `_ Note that this URL includes the encoded primary key of the record. Here's that same page as JSON: -`../people/uk.org.publicwhip%2Fperson%2F10001.json `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001.json `_ From 89c8ca0f3ff51fcbf5f710c529bc7a3552da0731 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:32:55 -0800 Subject: [PATCH 1169/1866] Fix for round_trip_load() YAML error, refs #2219 --- docs/metadata_doc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 3dc5b5f8..a8f13414 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,7 +1,7 @@ import json import textwrap from yaml import safe_dump -from ruamel.yaml import round_trip_load +from ruamel.yaml import YAML def metadata_example(cog, data=None, yaml=None): @@ -11,8 +11,7 @@ def metadata_example(cog, data=None, yaml=None): if yaml: # dedent it first yaml = textwrap.dedent(yaml).strip() - # round_trip_load to preserve key order: - data = round_trip_load(yaml) + data = YAML().load(yaml) output_yaml = yaml else: output_yaml = safe_dump(data, sort_keys=False) @@ -27,8 +26,7 @@ def metadata_example(cog, data=None, yaml=None): def config_example(cog, input): if type(input) is str: - # round_trip_load to preserve key order: - data = round_trip_load(input) + data = YAML().load(input) output_yaml = input else: data = input From 4284c74bc133ab494bf4b6dcd4a20b97b05ebb83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:51:03 -0800 Subject: [PATCH 1170/1866] db.execute_isolated_fn() method (#2220) Closes #2218 --- datasette/database.py | 61 ++++++++++++++++++++++++------ docs/internals.rst | 19 +++++++++- tests/test_internals_database.py | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index cb01301e..f2c980d7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -159,6 +159,26 @@ class Database: kwargs["count"] = count return results + async def execute_isolated_fn(self, fn): + # Open a new connection just for the duration of this function + # blocking the write queue to avoid any writes occurring during it + if self.ds.executor is None: + # non-threaded mode + isolated_connection = self.connect(write=True) + try: + result = fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + return result + else: + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) + async def execute_write_fn(self, fn, block=True): if self.ds.executor is None: # non-threaded mode @@ -166,9 +186,10 @@ class Database: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) return fn(self._write_connection) + else: + return await self._send_to_write_thread(fn, block) - # threaded mode - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -176,8 +197,9 @@ class Database: target=self._execute_writes, daemon=True ) self._write_thread.start() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue)) + self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -202,12 +224,28 @@ class Database: if conn_exception is not None: result = conn_exception else: - try: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e + if task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + else: + try: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): @@ -515,12 +553,13 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") - def __init__(self, fn, task_id, reply_queue): + def __init__(self, fn, task_id, reply_queue, isolated_connection): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue + self.isolated_connection = isolated_connection class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index 649ca35d..d269bc7d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1017,7 +1017,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() ` but executes the provided function in an entirely isolated SQLite connection, which is opened, used and then closed again in a single call to this method. + +The :ref:`prepare_connection() ` plugin hook is not executed against this connection. + +This allows plugins to execute database operations that might conflict with how database connections are usually configured. For example, running a ``VACUUM`` operation while bypassing any restrictions placed by the `datasette-sqlite-authorizer `__ plugin. + +Plugins can also use this method to load potentially dangerous SQLite extensions, use them to perform an operation and then have them safely unloaded at the end of the call, without risk of exposing them to other connections. + +Functions run using ``execute_isolated_fn()`` share the same queue as ``execute_write_fn()``, which guarantees that no writes can be executed at the same time as the isolated function is executing. + +The return value of the function will be returned by this method. Any exceptions raised by the function will be raised out of the ``await`` line as well. + .. _database_close: db.close() diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 647ae7bd..e0511100 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ +from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 from datasette.utils import Column @@ -519,6 +520,70 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +def table_exists(conn, name): + return bool( + conn.execute( + """ + with all_tables as ( + select name from sqlite_master where type = 'table' + union all + select name from temp.sqlite_master where type = 'table' + ) + select 1 from all_tables where name = ? + """, + (name,), + ).fetchall(), + ) + + +def table_exists_checker(name): + def inner(conn): + return table_exists(conn, name) + + return inner + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_threads", (False, True)) +async def test_execute_isolated(db, disable_threads): + if disable_threads: + ds = Datasette(memory=True, settings={"num_sql_threads": 0}) + db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) + + # Create temporary table in write + await db.execute_write( + "create temporary table created_by_write (id integer primary key)" + ) + # Should stay visible to write connection + assert await db.execute_write_fn(table_exists_checker("created_by_write")) + + def create_shared_table(conn): + conn.execute("create table shared (id integer primary key)") + # And a temporary table that should not continue to exist + conn.execute( + "create temporary table created_by_isolated (id integer primary key)" + ) + assert table_exists(conn, "created_by_isolated") + # Also confirm that created_by_write does not exist + return table_exists(conn, "created_by_write") + + # shared should not exist + assert not await db.execute_fn(table_exists_checker("shared")) + + # Create it using isolated + created_by_write_exists = await db.execute_isolated_fn(create_shared_table) + assert not created_by_write_exists + + # shared SHOULD exist now + assert await db.execute_fn(table_exists_checker("shared")) + + # created_by_isolated should not exist, even in write connection + assert not await db.execute_write_fn(table_exists_checker("created_by_isolated")) + + # ... and a second call to isolated should not see that connection either + assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) + + @pytest.mark.asyncio async def test_mtime_ns(db): assert isinstance(db.mtime_ns, int) From 978249beda1a3e7185f61000b0dd57018541c511 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:07:42 -0800 Subject: [PATCH 1171/1866] Removed rogue print("max_csv_mb") Found this while working on #2214 --- datasette/views/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 0080b33c..db08557e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -484,7 +484,6 @@ async def stream_csv(datasette, fetch_data, request, database): async def stream_fn(r): nonlocal data, trace - print("max_csv_mb", datasette.setting("max_csv_mb")) limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb")) if trace: await limited_writer.write(preamble) From 872dae1e1a1511e2edfb9d7ddf6ea5096c11d5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:08:11 -0800 Subject: [PATCH 1172/1866] Fix for CSV labels=on missing foreign key bug, closes #2214 --- datasette/views/base.py | 14 ++++++++------ tests/test_csv.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index db08557e..e59fd683 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -553,16 +553,18 @@ async def stream_csv(datasette, fetch_data, request, database): if cell is None: new_row.extend(("", "")) else: - assert isinstance(cell, dict) - new_row.append(cell["value"]) - new_row.append(cell["label"]) + if not isinstance(cell, dict): + new_row.extend((cell, "")) + else: + new_row.append(cell["value"]) + new_row.append(cell["label"]) else: new_row.append(cell) await writer.writerow(new_row) - except Exception as e: - sys.stderr.write("Caught this error: {}\n".format(e)) + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() - await r.write(str(e)) + await r.write(str(ex)) return await limited_writer.write(postamble) diff --git a/tests/test_csv.py b/tests/test_csv.py index ed83d685..9f772f89 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,3 +1,4 @@ +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup import pytest from .fixtures import ( # noqa @@ -95,6 +96,40 @@ async def test_table_csv_with_nullable_labels(ds_client): assert response.text == EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV +@pytest.mark.asyncio +async def test_table_csv_with_invalid_labels(): + # https://github.com/simonw/datasette/issues/2214 + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("db_2214") + await db.execute_write_script( + """ + create table t1 (id integer primary key, name text); + insert into t1 (id, name) values (1, 'one'); + insert into t1 (id, name) values (2, 'two'); + create table t2 (textid text primary key, name text); + insert into t2 (textid, name) values ('a', 'alpha'); + insert into t2 (textid, name) values ('b', 'beta'); + create table if not exists maintable ( + id integer primary key, + fk_integer integer references t1(id), + fk_text text references t2(textid) + ); + insert into maintable (id, fk_integer, fk_text) values (1, 1, 'a'); + insert into maintable (id, fk_integer, fk_text) values (2, 3, 'b'); -- invalid fk_integer + insert into maintable (id, fk_integer, fk_text) values (3, 2, 'c'); -- invalid fk_text + """ + ) + response = await ds.client.get("/db_2214/maintable.csv?_labels=1") + assert response.status_code == 200 + assert response.text == ( + "id,fk_integer,fk_integer_label,fk_text,fk_text_label\r\n" + "1,1,one,a,alpha\r\n" + "2,3,,b,beta\r\n" + "3,2,two,c,\r\n" + ) + + @pytest.mark.asyncio async def test_table_csv_blob_columns(ds_client): response = await ds_client.get("/fixtures/binary_data.csv") From 45b88f2056e0a4da204b50f5e17ba953fcb51865 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:14:50 -0800 Subject: [PATCH 1173/1866] Release notes from 0.64.6, refs #2214 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f2f17a50..af3d2a0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_6: + +0.64.6 (2023-12-22) +------------------- + +- Fixed a bug where CSV export with expanded labels could fail if a foreign key reference did not correctly resolve. (:issue:`2214`) + .. _v0_64_5: 0.64.5 (2023-10-08) From c7a4706bcc0d6736533b91437e54a8af9226a10a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 14:33:23 -0800 Subject: [PATCH 1174/1866] jinja2_environment_from_request() plugin hook Closes #2225 --- datasette/app.py | 49 +++++++++++++++++++++-------------- datasette/handle_exception.py | 3 ++- datasette/hookspecs.py | 5 ++++ datasette/views/base.py | 3 ++- datasette/views/database.py | 6 +++-- datasette/views/table.py | 3 ++- docs/plugin_hooks.rst | 42 ++++++++++++++++++++++++++++++ tests/test_plugins.py | 42 +++++++++++++++++++++++++++++- 8 files changed, 128 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f33865e4..482cebb4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -420,21 +420,31 @@ class Datasette: ), ] ) - self.jinja_env = Environment( + environment = Environment( loader=template_loader, autoescape=True, enable_async=True, # undefined=StrictUndefined, ) - self.jinja_env.filters["escape_css_string"] = escape_css_string - self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus - self.jinja_env.filters["escape_sqlite"] = escape_sqlite - self.jinja_env.filters["to_css_class"] = to_css_class + environment.filters["escape_css_string"] = escape_css_string + environment.filters["quote_plus"] = urllib.parse.quote_plus + self._jinja_env = environment + environment.filters["escape_sqlite"] = escape_sqlite + environment.filters["to_css_class"] = to_css_class self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_jinja_environment(self, request: Request = None) -> Environment: + environment = self._jinja_env + if request: + for environment in pm.hook.jinja2_environment_from_request( + datasette=self, request=request, env=environment + ): + pass + return environment + def get_permission(self, name_or_abbr: str) -> "Permission": """ Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. @@ -514,7 +524,7 @@ class Datasette: abbrs[p.abbr] = p self.permissions[p.name] = p for hook in pm.hook.prepare_jinja2_environment( - env=self.jinja_env, datasette=self + env=self._jinja_env, datasette=self ): await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): @@ -1218,7 +1228,7 @@ class Datasette: else: if isinstance(templates, str): templates = [templates] - template = self.jinja_env.select_template(templates) + template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): context = dataclasses.asdict(context) body_scripts = [] @@ -1568,16 +1578,6 @@ class DatasetteRouter: def __init__(self, datasette, routes): self.ds = datasette self.routes = routes or [] - # Build a list of pages/blah/{name}.html matching expressions - pattern_templates = [ - filepath - for filepath in self.ds.jinja_env.list_templates() - if "{" in filepath and filepath.startswith("pages/") - ] - self.page_routes = [ - (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) - for filepath in pattern_templates - ] async def __call__(self, scope, receive, send): # Because we care about "foo/bar" v.s. "foo%2Fbar" we decode raw_path ourselves @@ -1677,13 +1677,24 @@ class DatasetteRouter: route_path = request.scope.get("route_path", request.scope["path"]) # Jinja requires template names to use "/" even on Windows template_name = "pages" + route_path + ".html" + # Build a list of pages/blah/{name}.html matching expressions + environment = self.ds.get_jinja_environment(request) + pattern_templates = [ + filepath + for filepath in environment.list_templates() + if "{" in filepath and filepath.startswith("pages/") + ] + page_routes = [ + (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) + for filepath in pattern_templates + ] try: - template = self.ds.jinja_env.select_template([template_name]) + template = environment.select_template([template_name]) except TemplateNotFound: template = None if template is None: # Try for a pages/blah/{name}.html template match - for regex, wildcard_template in self.page_routes: + for regex, wildcard_template in page_routes: match = regex.match(route_path) if match is not None: context.update(match.groupdict()) diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 8b7e83e3..bef6b4ee 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -57,7 +57,8 @@ def handle_exception(datasette, request, exception): if request.path.split("?")[0].endswith(".json"): return Response.json(info, status=status, headers=headers) else: - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) return Response.html( await template.render_async( dict( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 9069927b..b6975dce 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -99,6 +99,11 @@ def actors_from_ids(datasette, actor_ids): """Returns a dictionary mapping those IDs to actor dictionaries""" +@hookspec +def jinja2_environment_from_request(datasette, request, env): + """Return a Jinja2 environment based on the incoming request""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/datasette/views/base.py b/datasette/views/base.py index e59fd683..bdc1e9cf 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -143,7 +143,8 @@ class BaseView: async def render(self, templates, request, context=None): context = context or {} - template = self.ds.jinja_env.select_template(templates) + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) template_context = { **context, **{ diff --git a/datasette/views/database.py b/datasette/views/database.py index 9ba5ce94..03e70379 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -143,7 +143,8 @@ class DatabaseView(View): datasette.urls.path(path_with_format(request=request, format="json")), ) templates = (f"database-{to_css_class(database)}.html", "database.html") - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) context = { **json_data, "database_color": db.color, @@ -594,7 +595,8 @@ class QueryView(View): f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/datasette/views/table.py b/datasette/views/table.py index 4f4baeed..7ee5d6bf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -806,7 +806,8 @@ async def table_view_traced(datasette, request): f"table-{to_css_class(resolved.db.name)}-{to_css_class(resolved.table)}.html", "table.html", ] - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index eb6bf4ae..f67d15d6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1128,6 +1128,48 @@ These IDs could be integers or strings, depending on how the actors used by the Example: `datasette-remote-actors `_ +.. _plugin_hook_jinja2_environment_from_request: + +jinja2_environment_from_request(datasette, request, env) +-------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + A Datasette instance. + +``request`` - :ref:`internals_request` or ``None`` + The current HTTP request, if one is available. + +``env`` - ``Environment`` + The Jinja2 environment that will be used to render the current page. + +This hook can be used to return a customized `Jinja environment `__ based on the incoming request. + +If you want to run a single Datasette instance that serves different content for different domains, you can do so like this: + +.. code-block:: python + + from datasette import hookimpl + from jinja2 import ChoiceLoader, FileSystemLoader + + + @hookimpl + def jinja2_environment_from_request(request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader( + "/mnt/niche-museums/templates" + ), + env.loader, + ] + ), + enable_async=True, + ) + return env + +This uses the Jinja `overlay() method `__ to create a new environment identical to the default environment except for having a different template loader, which first looks in the ``/mnt/niche-museums/templates`` directory before falling back on the default loader. + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 82e2f7f1..bdd4ba49 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -16,6 +16,7 @@ from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 from datasette.utils import CustomRow, StartupError from jinja2.environment import Template +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import importlib import json @@ -563,7 +564,8 @@ async def test_hook_register_output_renderer_can_render(ds_client): async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" await ds_client.ds.invoke_startup() - template = ds_client.ds.jinja_env.from_string( + environment = ds_client.ds.get_jinja_environment(None) + template = environment.from_string( "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", {"a": 3412341, "b": 5}, ) @@ -1294,3 +1296,41 @@ async def test_plugin_is_installed(): finally: pm.unregister(name="DummyPlugin") + + +@pytest.mark.asyncio +async def test_hook_jinja2_environment_from_request(tmpdir): + templates = pathlib.Path(tmpdir / "templates") + templates.mkdir() + (templates / "index.html").write_text("Hello museums!", "utf-8") + + class EnvironmentPlugin: + @hookimpl + def jinja2_environment_from_request(self, request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader(str(templates)), + env.loader, + ] + ), + enable_async=True, + ) + return env + + datasette = Datasette(memory=True) + + try: + pm.register(EnvironmentPlugin(), name="EnvironmentPlugin") + response = await datasette.client.get("/") + assert response.status_code == 200 + assert "Hello museums!" not in response.text + # Try again with the hostname + response2 = await datasette.client.get( + "/", headers={"host": "www.niche-museums.com"} + ) + assert response2.status_code == 200 + assert "Hello museums!" in response2.text + finally: + pm.unregister(name="EnvironmentPlugin") From 1fc76fee6268c21003c0fe730cc8e93210ce6bb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 16:59:25 -0800 Subject: [PATCH 1175/1866] 1.0a8.dev1 version number Not going to release this to PyPI but I will build my own wheel of it --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 55e2cd42..75d44727 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a7" +__version__ = "1.0a8.dev1" __version_info__ = tuple(__version__.split(".")) From 0b2c6a7ebd4fd540d9bdfb169c621452d280e608 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:12:57 -0800 Subject: [PATCH 1176/1866] Fix for ?_extra=columns bug, closes #2230 Also refs #262 - started a test suite for extras. --- datasette/renderer.py | 2 +- tests/test_table_api.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 224031a7..a446e69d 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -68,7 +68,7 @@ def json_renderer(request, args, data, error, truncated=None): elif shape in ("objects", "object", "array"): columns = data.get("columns") rows = data.get("rows") - if rows and columns: + if rows and columns and not isinstance(rows[0], dict): data["rows"] = [dict(zip(columns, row)) for row in rows] if shape == "object": shape_error = None diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 5dbb8b8f..ae4fdb17 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1362,3 +1362,27 @@ async def test_col_nocol_errors(ds_client, path, expected_error): response = await ds_client.get(path) assert response.status_code == 400 assert response.json()["error"] == expected_error + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra,expected_json", + ( + ( + "columns", + { + "ok": True, + "next": None, + "columns": ["id", "content", "content2"], + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + }, + ), + ), +) +async def test_table_extras(ds_client, extra, expected_json): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=" + extra + ) + assert response.status_code == 200 + assert response.json() == expected_json From 2ff4d4a60a348c143f79d63c48c329ffd0c1f02f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:13:53 -0800 Subject: [PATCH 1177/1866] Test for ?_extra=count, refs #262 --- tests/test_table_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ae4fdb17..bde7a38e 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1378,6 +1378,16 @@ async def test_col_nocol_errors(ds_client, path, expected_error): "truncated": False, }, ), + ( + "count", + { + "ok": True, + "next": None, + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + "count": 1, + }, + ), ), ) async def test_table_extras(ds_client, extra, expected_json): From 48148e66a846d585e08ec6ab4ae3da8e60d55ab5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 10:42:36 -0800 Subject: [PATCH 1178/1866] Link from actors_from_ids plugin hook docs to datasette.actors_from_ids() --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f67d15d6..9115c3df 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1086,6 +1086,8 @@ The hook must return a dictionary that maps the incoming actor IDs to their full Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. +The :ref:`await datasette.actors_from_ids(actor_ids) ` internal method can be used to look up actors from their IDs. It will dispatch to the first plugin that implements this hook. + Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. From 7506a89be0d1c97632bed47635eb90f92815d6c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:04:34 -0800 Subject: [PATCH 1179/1866] Docs on datasette.client for tests, closes #1830 Also covers ds.client.actor_cookie() helper --- docs/testing_plugins.rst | 28 +++++++++++++++++++++++++++ tests/test_docs.py | 41 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 6d2097ad..e10514c6 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,34 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_datasette_client: + +Using datasette.client in tests +------------------------------- + +The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. + +I simple test looks like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_homepage -- + :end-before: # -- end test_homepage -- + +Or for a JSON API: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_actor_is_null -- + :end-before: # -- end test_actor_is_null -- + +To make requests as an authenticated actor, create a signed ``ds_cookie`` using the ``datasette.client.actor_cookie()`` helper function and pass it in ``cookies=`` like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_signed_cookie_actor -- + :end-before: # -- end test_signed_cookie_actor -- + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/test_docs.py b/tests/test_docs.py index e9b813fe..fdd44788 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,9 +1,8 @@ """ Tests to ensure certain things are documented. """ -from click.testing import CliRunner from datasette import app, utils -from datasette.cli import cli +from datasette.app import Datasette from datasette.filters import Filters from pathlib import Path import pytest @@ -102,3 +101,41 @@ def documented_fns(): @pytest.mark.parametrize("fn", utils.functions_marked_as_documented) def test_functions_marked_with_documented_are_documented(documented_fns, fn): assert fn.__name__ in documented_fns + + +# Tests for testing_plugins.rst documentation + + +# -- start test_homepage -- +@pytest.mark.asyncio +async def test_homepage(): + ds = Datasette(memory=True) + response = await ds.client.get("/") + html = response.text + assert "

    " in html + + +# -- end test_homepage -- + + +# -- start test_actor_is_null -- +@pytest.mark.asyncio +async def test_actor_is_null(): + ds = Datasette(memory=True) + response = await ds.client.get("/-/actor.json") + assert response.json() == {"actor": None} + + +# -- end test_actor_is_null -- + + +# -- start test_signed_cookie_actor -- +@pytest.mark.asyncio +async def test_signed_cookie_actor(): + ds = Datasette(memory=True) + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/-/actor.json", cookies=cookies) + assert response.json() == {"actor": {"id": "root"}} + + +# -- end test_signed_cookie_actor -- From 0f63cb83ed31753a9bd9ec5cc71de16906767337 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:08:52 -0800 Subject: [PATCH 1180/1866] Typo fix --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index e10514c6..33ac4b22 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -89,7 +89,7 @@ Using datasette.client in tests The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. -I simple test looks like this: +A simple test looks like this: .. literalinclude:: ../tests/test_docs.py :language: python From a25bf6bea789c409580386f77b7440ff525d09b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 14:10:40 -0800 Subject: [PATCH 1181/1866] fmt: off to fix problem with Black, closes #2231 --- tests/test_docs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index fdd44788..17c01a0b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -105,7 +105,7 @@ def test_functions_marked_with_documented_are_documented(documented_fns, fn): # Tests for testing_plugins.rst documentation - +# fmt: off # -- start test_homepage -- @pytest.mark.asyncio async def test_homepage(): @@ -113,8 +113,6 @@ async def test_homepage(): response = await ds.client.get("/") html = response.text assert "

    " in html - - # -- end test_homepage -- @@ -124,8 +122,6 @@ async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") assert response.json() == {"actor": None} - - # -- end test_actor_is_null -- @@ -136,6 +132,4 @@ async def test_signed_cookie_actor(): cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) assert response.json() == {"actor": {"id": "root"}} - - # -- end test_signed_cookie_actor -- From 7a5adb592ae6674a2058639c66e85eb1b49448fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jan 2024 14:12:14 -0800 Subject: [PATCH 1182/1866] Docs on temporary plugins in fixtures, closes #2234 --- docs/testing_plugins.rst | 16 ++++++++++++++++ tests/test_docs_plugins.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/test_docs_plugins.py diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 33ac4b22..f1363fb4 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -313,3 +313,19 @@ When writing tests for plugins you may find it useful to register a test plugin assert response.status_code == 500 finally: pm.unregister(name="undo") + +To reuse the same temporary plugin in multiple tests, you can register it inside a fixture in your ``conftest.py`` file like this: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_fixture -- + :end-before: # -- end datasette_with_plugin_fixture -- + +Note the ``yield`` statement here - this ensures that the ``finally:`` block that unregisters the plugin is executed only after the test function itself has completed. + +Then in a test: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_test -- + :end-before: # -- end datasette_with_plugin_test -- diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py new file mode 100644 index 00000000..92b4514c --- /dev/null +++ b/tests/test_docs_plugins.py @@ -0,0 +1,34 @@ +# fmt: off +# -- start datasette_with_plugin_fixture -- +from datasette import hookimpl +from datasette.app import Datasette +from datasette.plugins import pm +import pytest +import pytest_asyncio + + +@pytest_asyncio.fixture +async def datasette_with_plugin(): + class TestPlugin: + __name__ = "TestPlugin" + + @hookimpl + def register_routes(self): + return [ + (r"^/error$", lambda: 1 / 0), + ] + + pm.register(TestPlugin(), name="undo") + try: + yield Datasette() + finally: + pm.unregister(name="undo") +# -- end datasette_with_plugin_fixture -- + + +# -- start datasette_with_plugin_test -- +@pytest.mark.asyncio +async def test_error(datasette_with_plugin): + response = await datasette_with_plugin.client.get("/error") + assert response.status_code == 500 +# -- end datasette_with_plugin_test -- From c3caf36af7db79336a5c8e697b2374e90e34ff5d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:54:03 -0800 Subject: [PATCH 1183/1866] Template slot family of plugin hooks - top_homepage() and others New plugin hooks: top_homepage top_database top_table top_row top_query top_canned_query New datasette.utils.make_slot_function() Closes #1191 --- datasette/hookspecs.py | 30 ++++++++ datasette/templates/database.html | 2 + datasette/templates/index.html | 2 + datasette/templates/query.html | 2 + datasette/templates/row.html | 2 + datasette/templates/table.html | 2 + datasette/utils/__init__.py | 17 +++++ datasette/views/database.py | 21 +++++- datasette/views/index.py | 9 ++- datasette/views/row.py | 12 ++- datasette/views/table.py | 8 ++ docs/plugin_hooks.rst | 119 ++++++++++++++++++++++++++++++ tests/test_docs.py | 4 +- tests/test_plugins.py | 101 +++++++++++++++++++++++++ 14 files changed, 324 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index b6975dce..2f4c6027 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -158,3 +158,33 @@ def skip_csrf(datasette, scope): @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" + + +@hookspec +def top_homepage(datasette, request): + """HTML to include at the top of the homepage""" + + +@hookspec +def top_database(datasette, request, database): + """HTML to include at the top of the database page""" + + +@hookspec +def top_table(datasette, request, database, table): + """HTML to include at the top of the table page""" + + +@hookspec +def top_row(datasette, request, database, table, row): + """HTML to include at the top of the row page""" + + +@hookspec +def top_query(datasette, request, database, sql): + """HTML to include at the top of the query results page""" + + +@hookspec +def top_canned_query(datasette, request, database, query_name): + """HTML to include at the top of the canned query page""" diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 3d4dae07..4b125a44 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -34,6 +34,8 @@

    {% endif %}
    +{{ top_database() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index 06e09635..203abca8 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -7,6 +7,8 @@ {% block content %}

    {{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}

    +{{ top_homepage() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% for database in databases %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b8f06f84..1815e592 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -30,6 +30,8 @@

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 4d179a85..6d4b996e 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,6 +22,8 @@ {% block content %}

    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

    +{{ top_row() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}

    diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 88580e52..5aee6319 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -45,6 +45,8 @@
    {% endif %}
    +{{ top_table() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if metadata.get("columns") %} diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0f449b89..8914c043 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1283,3 +1283,20 @@ def fail_if_plugins_in_metadata(metadata: dict, filename=None): f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' ) return metadata + + +def make_slot_function(name, datasette, request, **kwargs): + from datasette.plugins import pm + + method = getattr(pm.hook, name, None) + assert method is not None, "No hook found for {}".format(name) + + async def inner(): + html_bits = [] + for hook in method(datasette=datasette, request=request, **kwargs): + html = await await_me_maybe(hook) + if html is not None: + html_bits.append(html) + return markupsafe.Markup("".join(html_bits)) + + return inner diff --git a/datasette/views/database.py b/datasette/views/database.py index 03e70379..caeb4e46 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from typing import Callable from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -18,6 +17,7 @@ from datasette.utils import ( call_with_supported_arguments, derive_named_parameters, format_bytes, + make_slot_function, tilde_decode, to_css_class, validate_sql_select, @@ -161,6 +161,9 @@ class DatabaseView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + "top_database": make_slot_function( + "top_database", datasette, request, database=database + ), } return Response.html( await datasette.render_template( @@ -246,6 +249,12 @@ class QueryContext: "help": "List of templates that were considered for rendering this page" } ) + top_query: callable = field( + metadata={"help": "Callable to render the top_query slot"} + ) + top_canned_query: callable = field( + metadata={"help": "Callable to render the top_canned_query slot"} + ) async def get_tables(datasette, request, db): @@ -727,6 +736,16 @@ class QueryView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_query=make_slot_function( + "top_query", datasette, request, database=database, sql=sql + ), + top_canned_query=make_slot_function( + "top_canned_query", + datasette, + request, + database=database, + query_name=canned_query["name"] if canned_query else None, + ), ), request=request, view_name="database", diff --git a/datasette/views/index.py b/datasette/views/index.py index 95b29302..595cf234 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,10 +1,12 @@ -import hashlib import json -from datasette.utils import add_cors_headers, CustomJSONEncoder +from datasette.plugins import pm +from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ +from markupsafe import Markup + from .base import BaseView @@ -142,5 +144,8 @@ class IndexView(BaseView): "private": not await self.ds.permission_allowed( None, "view-instance" ), + "top_homepage": make_slot_function( + "top_homepage", self.ds, request + ), }, ) diff --git a/datasette/views/row.py b/datasette/views/row.py index 8f07a662..ce877753 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -2,11 +2,9 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from .base import DataView, BaseView, _error from datasette.utils import ( - tilde_decode, - urlsafe_components, + make_slot_function, to_css_class, escape_sqlite, - row_sql_params_pks, ) import json import sqlite_utils @@ -73,6 +71,14 @@ class RowView(DataView): .get(database, {}) .get("tables", {}) .get(table, {}), + "top_row": make_slot_function( + "top_row", + self.ds, + request, + database=resolved.db.name, + table=resolved.table, + row=rows[0], + ), } data = { diff --git a/datasette/views/table.py b/datasette/views/table.py index 7ee5d6bf..be7479f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -17,6 +17,7 @@ from datasette.utils import ( append_querystring, compound_keys_after_sql, format_bytes, + make_slot_function, tilde_encode, escape_sqlite, filters_should_redirect, @@ -842,6 +843,13 @@ async def table_view_traced(datasette, request): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), ), request=request, view_name="table", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9115c3df..ce648ba7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1641,3 +1641,122 @@ This hook is responsible for returning a dictionary corresponding to Datasette : return metadata Example: `datasette-remote-metadata plugin `__ + +.. _plugin_hook_slots: + +Template slots +-------------- + +The following set of plugin hooks can be used to return extra HTML content that will be inserted into the corresponding page, directly below the ``

    `` heading. + +Multiple plugins can contribute content here. The order in which it is displayed can be controlled using Pluggy's `call time order options `__. + +Each of these plugin hooks can return either a string or an awaitable function that returns a string. + +.. _plugin_hook_top_homepage: + +top_homepage(datasette, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +Returns HTML to be displayed at the top of the Datasette homepage. + +.. _plugin_hook_top_database: + +top_database(datasette, request, database) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +Returns HTML to be displayed at the top of the database page. + +.. _plugin_hook_top_table: + +top_table(datasette, request, database, table) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +Returns HTML to be displayed at the top of the table page. + +.. _plugin_hook_top_row: + +top_row(datasette, request, database, table, row) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``row`` - ``sqlite.Row`` + The SQLite row object being displayed. + +Returns HTML to be displayed at the top of the row page. + +.. _plugin_hook_top_query: + +top_query(datasette, request, database, sql) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``sql`` - string + The SQL query. + +Returns HTML to be displayed at the top of the query results page. + +.. _plugin_hook_top_canned_query: + +top_canned_query(datasette, request, database, query_name) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``query_name`` - string + The name of the canned query. + +Returns HTML to be displayed at the top of the canned query page. diff --git a/tests/test_docs.py b/tests/test_docs.py index 17c01a0b..0a803861 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -41,7 +41,9 @@ def plugin_hooks_content(): "plugin", [name for name in dir(app.pm.hook) if not name.startswith("_")] ) def test_plugin_hooks_are_documented(plugin, plugin_hooks_content): - headings = get_headings(plugin_hooks_content, "-") + headings = set() + headings.update(get_headings(plugin_hooks_content, "-")) + headings.update(get_headings(plugin_hooks_content, "~")) assert plugin in headings hook_caller = getattr(app.pm.hook, plugin) arg_names = [a for a in hook_caller.spec.argnames if a != "__multicall__"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index bdd4ba49..784c460a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1334,3 +1334,104 @@ async def test_hook_jinja2_environment_from_request(tmpdir): assert "Hello museums!" in response2.text finally: pm.unregister(name="EnvironmentPlugin") + + +class SlotPlugin: + __name__ = "SlotPlugin" + + @hookimpl + def top_homepage(self, request): + return "Xtop_homepage:" + request.args["z"] + + @hookimpl + def top_database(self, request, database): + async def inner(): + return "Xtop_database:{}:{}".format(database, request.args["z"]) + + return inner + + @hookimpl + def top_table(self, request, database, table): + return "Xtop_table:{}:{}:{}".format(database, table, request.args["z"]) + + @hookimpl + def top_row(self, request, database, table, row): + return "Xtop_row:{}:{}:{}:{}".format( + database, table, row["name"], request.args["z"] + ) + + @hookimpl + def top_query(self, request, database, sql): + return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) + + @hookimpl + def top_canned_query(self, request, database, query_name): + return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + + +@pytest.mark.asyncio +async def test_hook_top_homepage(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/?z=foo") + assert response.status_code == 200 + assert "Xtop_homepage:foo" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_database(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/_memory?z=bar") + assert response.status_code == 200 + assert "Xtop_database:_memory:bar" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_table(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facetable?z=baz") + assert response.status_code == 200 + assert "Xtop_table:fixtures:facetable:baz" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_row(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facet_cities/1?z=bax") + assert response.status_code == 200 + assert "Xtop_row:fixtures:facet_cities:San Francisco:bax" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures?sql=select+1&z=x") + assert response.status_code == 200 + assert "Xtop_query:fixtures:select 1:x" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_canned_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/from_hook?z=xyz") + assert response.status_code == 200 + assert "Xtop_query:fixtures:from_hook:xyz" in response.text + finally: + pm.unregister(name="SlotPlugin") From 5c64af69363100a3c35e6b131efe1f741bbde661 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:55:26 -0800 Subject: [PATCH 1184/1866] Upgrade to latest Black, closes #2239 --- datasette/filters.py | 14 ++++++------ datasette/utils/__init__.py | 14 ++++++------ datasette/utils/shutil_backport.py | 1 + datasette/views/database.py | 22 +++++++++++-------- datasette/views/table.py | 30 ++++++++++++++------------ setup.py | 2 +- tests/plugins/my_plugin.py | 6 +++--- tests/test_api_write.py | 8 ++++--- tests/test_cli.py | 8 ++++--- tests/test_docs.py | 1 + tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 1 + tests/test_permissions.py | 8 ++++--- tests/test_plugins.py | 34 ++++++++++++++++-------------- tests/test_table_api.py | 8 ++++--- tests/test_utils.py | 1 + 16 files changed, 93 insertions(+), 66 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 5ea3488b..73eea857 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -80,9 +80,9 @@ def search_filters(request, database, table, datasette): "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( fts_table=escape_sqlite(fts_table), fts_pk=escape_sqlite(fts_pk), - match_clause=":search" - if search_mode_raw - else "escape_fts(:search)", + match_clause=( + ":search" if search_mode_raw else "escape_fts(:search)" + ), ) ) human_descriptions.append(f'search matches "{search}"') @@ -99,9 +99,11 @@ def search_filters(request, database, table, datasette): "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), - match_clause=":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i), + match_clause=( + ":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i) + ), ) ) human_descriptions.append( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 8914c043..196e1682 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -402,9 +402,9 @@ def make_dockerfile( apt_get_extras = apt_get_extras_ if spatialite: apt_get_extras.extend(["python3-dev", "gcc", "libsqlite3-mod-spatialite"]) - environment_variables[ - "SQLITE_EXTENSIONS" - ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + environment_variables["SQLITE_EXTENSIONS"] = ( + "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + ) return """ FROM python:3.11.0-slim-bullseye COPY . /app @@ -416,9 +416,11 @@ RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT {port} EXPOSE {port} CMD {cmd}""".format( - apt_get_extras=APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) - if apt_get_extras - else "", + apt_get_extras=( + APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) + if apt_get_extras + else "" + ), environment_variables="\n".join( [ "ENV {} '{}'".format(key, value) diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index dbe22404..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -4,6 +4,7 @@ Backported from Python 3.8. This code is licensed under the Python License: https://github.com/python/cpython/blob/v3.8.3/LICENSE """ + import os from shutil import copy, copy2, copystat, Error diff --git a/datasette/views/database.py b/datasette/views/database.py index caeb4e46..eac01ab6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -126,9 +126,9 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "allow_execute_sql": allow_execute_sql, - "table_columns": await _table_columns(datasette, database) - if allow_execute_sql - else {}, + "table_columns": ( + await _table_columns(datasette, database) if allow_execute_sql else {} + ), } if format_ == "json": @@ -719,9 +719,11 @@ class QueryView(View): display_rows=await display_rows( datasette, database, request, rows, columns ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, + table_columns=( + await _table_columns(datasette, database) + if allow_execute_sql + else {} + ), columns=columns, renderers=renderers, url_csv=datasette.urls.path( @@ -1036,9 +1038,11 @@ async def display_rows(datasette, database, request, rows, columns): display_value = markupsafe.Markup( '<Binary: {:,} byte{}>'.format( blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) diff --git a/datasette/views/table.py b/datasette/views/table.py index be7479f8..2c5e3e13 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -236,9 +236,11 @@ async def display_columns_and_rows( path_from_row_pks(row, pks, not pks), column, ), - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) @@ -289,9 +291,9 @@ async def display_columns_and_rows( "column": column, "value": display_value, "raw": value, - "value_type": "none" - if value is None - else str(type(value).__name__), + "value_type": ( + "none" if value is None else str(type(value).__name__) + ), } ) cell_rows.append(Row(cells)) @@ -974,9 +976,9 @@ async def table_view_data( from_sql = "from {table_name} {where}".format( table_name=escape_sqlite(table_name), - where=("where {} ".format(" and ".join(where_clauses))) - if where_clauses - else "", + where=( + ("where {} ".format(" and ".join(where_clauses))) if where_clauses else "" + ), ) # Copy of params so we can mutate them later: from_sql_params = dict(**params) @@ -1040,10 +1042,12 @@ async def table_view_data( column=escape_sqlite(sort or sort_desc), op=">" if sort else "<", p=len(params), - extra_desc_only="" - if sort - else " or {column2} is null".format( - column2=escape_sqlite(sort or sort_desc) + extra_desc_only=( + "" + if sort + else " or {column2} is null".format( + column2=escape_sqlite(sort or sort_desc) + ) ), next_clauses=" and ".join(next_by_pk_clauses), ) diff --git a/setup.py b/setup.py index d09a9e3d..cd393368 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.9.1", + "black==24.1.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index eb70d9bd..9d1f86bc 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -39,9 +39,9 @@ def extra_css_urls(template, database, table, view_name, columns, request, datas "database": database, "table": table, "view_name": view_name, - "request_path": request.path - if request is not None - else None, + "request_path": ( + request.path if request is not None else None + ), "added": ( await datasette.get_database().execute("select 3 * 5") ).first()[0], diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f27d143f..1787e06f 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -279,9 +279,11 @@ async def test_insert_or_upsert_row_errors( json=input, headers={ "Authorization": "Bearer {}".format(token), - "Content-Type": "text/plain" - if special_case == "invalid_content_type" - else "application/json", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), }, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 213db416..080e8353 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -335,9 +335,11 @@ def test_serve_create(tmpdir): def test_serve_config(tmpdir, argument, format_): config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( - "settings:\n default_page_size: 5\n" - if format_ == "yaml" - else '{"settings": {"default_page_size": 5}}', + ( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}' + ), "utf-8", ) runner = CliRunner() diff --git a/tests/test_docs.py b/tests/test_docs.py index 0a803861..2a58d954 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,6 +1,7 @@ """ Tests to ensure certain things are documented. """ + from datasette import app, utils from datasette.app import Datasette from datasette.filters import Filters diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e0511100..dd68a6cb 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ + from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 428b259d..c30bb748 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,6 +1,7 @@ """ Tests for the datasette.app.Datasette class """ + import dataclasses from datasette import Forbidden, Context from datasette.app import Datasette, Database diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 933aa07b..9917b749 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -381,9 +381,11 @@ async def test_permissions_debug(ds_client): { "action": div.select_one(".check-action").text, # True = green tick, False = red cross, None = gray None - "result": None - if div.select(".check-result-no-opinion") - else bool(div.select(".check-result-true")), + "result": ( + None + if div.select(".check-result-no-opinion") + else bool(div.select(".check-result-true")) + ), "used_default": bool(div.select(".check-used-default")), } for div in check_divs diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 784c460a..5bfb6132 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1096,24 +1096,26 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - config={ - "plugins": { - "datasette-register-permissions": { - "permissions": [ - { - "name": "extra-from-metadata", - "abbr": "efm", - "description": "Extra from metadata", - "takes_database": False, - "takes_resource": False, - "default": True, - } - ] + config=( + { + "plugins": { + "datasette-register-permissions": { + "permissions": [ + { + "name": "extra-from-metadata", + "abbr": "efm", + "description": "Extra from metadata", + "takes_database": False, + "takes_resource": False, + "default": True, + } + ] + } } } - } - if extra_metadata - else None, + if extra_metadata + else None + ), plugins_dir=PLUGINS_DIR, ) await ds.invoke_startup() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index bde7a38e..58930950 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -305,9 +305,11 @@ async def test_paginate_compound_keys_with_extra_filters(ds_client): "_sort_desc=sortable_with_nulls", lambda row: ( 1 if row["sortable_with_nulls"] is None else 0, - -row["sortable_with_nulls"] - if row["sortable_with_nulls"] is not None - else 0, + ( + -row["sortable_with_nulls"] + if row["sortable_with_nulls"] is not None + else 0 + ), row["content"], ), "sorted by sortable_with_nulls descending", diff --git a/tests/test_utils.py b/tests/test_utils.py index 61392b8b..51577615 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ """ Tests for various datasette helper functions. """ + from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request From b8230694ff90f9a6cd4f5b7c47fd8a71c831ee1d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Tue, 30 Jan 2024 22:56:05 -0500 Subject: [PATCH 1185/1866] Set link to download db to nofollow --- datasette/templates/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 4b125a44..ee4dd705 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -97,7 +97,7 @@ {% endif %} {% if allow_download %} -

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    +

    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

    {% endif %} {% include "_codemirror_foot.html" %} From 04e8835297760416b50cc669ac6f45a7fd68170b Mon Sep 17 00:00:00 2001 From: gerrymanoim Date: Tue, 30 Jan 2024 22:56:32 -0500 Subject: [PATCH 1186/1866] Remove deprecated/unused args from setup.py (#2222) --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index cd393368..53206eae 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ setup( [console_scripts] datasette=datasette.cli:cli """, - setup_requires=["pytest-runner"], extras_require={ "docs": [ "Sphinx==7.2.6", @@ -93,7 +92,6 @@ setup( ], "rich": ["rich"], }, - tests_require=["datasette[test]"], classifiers=[ "Development Status :: 4 - Beta", "Framework :: Datasette", From 959e0202972f4d95088c4c1a9df6274108af8bfb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 20:40:18 -0800 Subject: [PATCH 1187/1866] Ran blacken-docs --- docs/internals.rst | 3 +-- docs/plugin_hooks.rst | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index d269bc7d..d8f86251 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -210,8 +210,7 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The secure=False, httponly=False, samesite="lax", - ): - ... + ): ... You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie ` for use with Datasette :ref:`authentication `: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ce648ba7..da69c6c9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -373,8 +373,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ about, about_url, api_key, - ): - ... + ): ... Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ From 890615b3f29dcf82a792f1a145b02dba784a5b63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:53:57 -0800 Subject: [PATCH 1188/1866] Bump the python-packages group with 1 update (#2241) Bumps the python-packages group with 1 update: [furo](https://github.com/pradyunsg/furo). Updates `furo` from 2023.9.10 to 2024.1.29 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.09.10...2024.01.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-major dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53206eae..b3915c42 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( extras_require={ "docs": [ "Sphinx==7.2.6", - "furo==2023.9.10", + "furo==2024.1.29", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", From bcc4f6bf1f14be6ef693f0b3fc9aa8a027977920 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:21:40 -0800 Subject: [PATCH 1189/1866] track_event() mechanism for analytics and plugins * Closes #2240 * Documentation for event plugin hooks, refs #2240 * Include example track_event plugin in docs, refs #2240 * Tests for track_event() and register_events() hooks, refs #2240 * Initial documentation for core events, refs #2240 * Internals documentation for datasette.track_event() --- datasette/__init__.py | 1 + datasette/app.py | 16 +++ datasette/events.py | 211 ++++++++++++++++++++++++++++++++++++ datasette/hookspecs.py | 10 ++ datasette/plugins.py | 1 + datasette/views/database.py | 6 + datasette/views/row.py | 20 ++++ datasette/views/special.py | 17 ++- datasette/views/table.py | 31 ++++++ docs/conf.py | 2 + docs/events.rst | 14 +++ docs/index.rst | 1 + docs/internals.rst | 20 ++++ docs/plugin_hooks.rst | 100 +++++++++++++++++ docs/plugins.rst | 9 ++ tests/conftest.py | 33 +++++- tests/test_api.py | 7 +- tests/test_api_write.py | 64 +++++++++++ tests/test_auth.py | 19 +++- tests/test_cli.py | 6 +- tests/test_plugins.py | 31 +++++- tests/utils.py | 5 + 22 files changed, 614 insertions(+), 10 deletions(-) create mode 100644 datasette/events.py create mode 100644 docs/events.rst diff --git a/datasette/__init__.py b/datasette/__init__.py index 271e09ad..47d2b4f6 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,5 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa +from datasette.events import Event # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/app.py b/datasette/app.py index 482cebb4..530f79bc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -34,6 +34,7 @@ from jinja2 import ( from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from .events import Event from .views import Context from .views.base import ureg from .views.database import database_download, DatabaseView, TableCreateView @@ -505,6 +506,14 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) + # Register permissions, but watch out for duplicate name/abbr names = {} abbrs = {} @@ -873,6 +882,13 @@ class Datasette: result = await await_me_maybe(result) return result + async def track_event(self, event: Event): + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) + for hook in pm.hook.track_event(datasette=self, event=event): + await await_me_maybe(hook) + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/events.py b/datasette/events.py new file mode 100644 index 00000000..96244779 --- /dev/null +++ b/datasette/events.py @@ -0,0 +1,211 @@ +from abc import ABC, abstractproperty +from dataclasses import asdict, dataclass, field +from datasette.hookspecs import hookimpl +from datetime import datetime, timezone +from typing import Optional + + +@dataclass +class Event(ABC): + @abstractproperty + def name(self): + pass + + created: datetime = field( + init=False, default_factory=lambda: datetime.now(timezone.utc) + ) + actor: Optional[dict] + + def properties(self): + properties = asdict(self) + properties.pop("actor", None) + properties.pop("created", None) + return properties + + +@dataclass +class LoginEvent(Event): + """ + Event name: ``login`` + + A user (represented by ``event.actor``) has logged in. + """ + + name = "login" + + +@dataclass +class LogoutEvent(Event): + """ + Event name: ``logout`` + + A user (represented by ``event.actor``) has logged out. + """ + + name = "logout" + + +@dataclass +class CreateTokenEvent(Event): + """ + Event name: ``create-token`` + + A user created an API token. + + :ivar expires_after: Number of seconds after which this token will expire. + :type expires_after: int or None + :ivar restrict_all: Restricted permissions for this token. + :type restrict_all: list + :ivar restrict_database: Restricted database permissions for this token. + :type restrict_database: dict + :ivar restrict_resource: Restricted resource permissions for this token. + :type restrict_resource: dict + """ + + name = "create-token" + expires_after: Optional[int] + restrict_all: list + restrict_database: dict + restrict_resource: dict + + +@dataclass +class CreateTableEvent(Event): + """ + Event name: ``create-table`` + + A new table has been created in the database. + + :ivar database: The name of the database where the table was created. + :type database: str + :ivar table: The name of the table that was created + :type table: str + :ivar schema: The SQL schema definition for the new table. + :type schema: str + """ + + name = "create-table" + database: str + table: str + schema: str + + +@dataclass +class DropTableEvent(Event): + """ + Event name: ``drop-table`` + + A table has been dropped from the database. + + :ivar database: The name of the database where the table was dropped. + :type database: str + :ivar table: The name of the table that was dropped + :type table: str + """ + + name = "drop-table" + database: str + table: str + + +@dataclass +class InsertRowsEvent(Event): + """ + Event name: ``insert-rows`` + + Rows were inserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + :ivar ignore: Was ignore set? + :type ignore: bool + :ivar replace: Was replace set? + :type replace: bool + """ + + name = "insert-rows" + database: str + table: str + num_rows: int + ignore: bool + replace: bool + + +@dataclass +class UpsertRowsEvent(Event): + """ + Event name: ``upsert-rows`` + + Rows were upserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + """ + + name = "upsert-rows" + database: str + table: str + num_rows: int + + +@dataclass +class UpdateRowEvent(Event): + """ + Event name: ``update-row`` + + A row was updated in a table. + + :ivar database: The name of the database where the row was updated. + :type database: str + :ivar table: The name of the table where the row was updated. + :type table: str + :ivar pks: The primary key values of the updated row. + """ + + name = "update-row" + database: str + table: str + pks: list + + +@dataclass +class DeleteRowEvent(Event): + """ + Event name: ``delete-row`` + + A row was deleted from a table. + + :ivar database: The name of the database where the row was deleted. + :type database: str + :ivar table: The name of the table where the row was deleted. + :type table: str + :ivar pks: The primary key values of the deleted row. + """ + + name = "delete-row" + database: str + table: str + pks: list + + +@hookimpl +def register_events(): + return [ + LoginEvent, + LogoutEvent, + CreateTableEvent, + CreateTokenEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, + UpdateRowEvent, + DeleteRowEvent, + ] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 2f4c6027..b473f398 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -160,6 +160,16 @@ def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" +@hookspec +def track_event(datasette, event): + """Respond to an event tracked by Datasette""" + + +@hookspec +def register_events(datasette): + """Return a list of Event subclasses to use with track_event()""" + + @hookspec def top_homepage(datasette, request): """HTML to include at the top of the homepage""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 1ed3747f..f7a1905f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -27,6 +27,7 @@ DEFAULT_PLUGINS = ( "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", + "datasette.events", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/views/database.py b/datasette/views/database.py index eac01ab6..6d17b16c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,6 +10,7 @@ import re import sqlite_utils import textwrap +from datasette.events import CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -969,6 +970,11 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) + ) return Response.json(details, status=201) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce877753..7b646641 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,5 +1,6 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted +from datasette.events import UpdateRowEvent, DeleteRowEvent from .base import DataView, BaseView, _error from datasette.utils import ( make_slot_function, @@ -200,6 +201,15 @@ class RowDeleteView(BaseView): except Exception as e: return _error([str(e)], 500) + await self.ds.track_event( + DeleteRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json({"ok": True}, status=200) @@ -246,4 +256,14 @@ class RowUpdateView(BaseView): ) rows = list(results.rows) result["row"] = dict(rows[0]) + + await self.ds.track_event( + UpdateRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json(result, status=200) diff --git a/datasette/views/special.py b/datasette/views/special.py index 849750bf..4088a1f9 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,4 +1,5 @@ import json +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( actor_matches_allow, @@ -80,9 +81,9 @@ class AuthTokenView(BaseView): if secrets.compare_digest(token, self.ds._root_token): self.ds._root_token = None response = Response.redirect(self.ds.urls.instance()) - response.set_cookie( - "ds_actor", self.ds.sign({"a": {"id": "root"}}, "actor") - ) + root_actor = {"id": "root"} + response.set_cookie("ds_actor", self.ds.sign({"a": root_actor}, "actor")) + await self.ds.track_event(LoginEvent(actor=root_actor)) return response else: raise Forbidden("Invalid token") @@ -105,6 +106,7 @@ class LogoutView(BaseView): response = Response.redirect(self.ds.urls.instance()) response.set_cookie("ds_actor", "", expires=0, max_age=0) self.ds.add_message(request, "You are now logged out", self.ds.WARNING) + await self.ds.track_event(LogoutEvent(actor=request.actor)) return response @@ -349,6 +351,15 @@ class CreateTokenView(BaseView): restrict_resource=restrict_resource, ) token_bits = self.ds.unsign(token[len("dstok_") :], namespace="token") + await self.ds.track_event( + CreateTokenEvent( + actor=request.actor, + expires_after=expires_after, + restrict_all=restrict_all, + restrict_database=restrict_database, + restrict_resource=restrict_resource, + ) + ) context = await self.shared(request) context.update({"errors": errors, "token": token, "token_bits": token_bits}) return await self.render(["create_token.html"], request, context) diff --git a/datasette/views/table.py b/datasette/views/table.py index 2c5e3e13..3b812c01 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,6 +8,7 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted +from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -467,6 +468,8 @@ class TableInsertView(BaseView): if errors: return _error(errors, 400) + num_rows = len(rows) + # No that we've passed pks to _validate_data it's safe to # fix the rowids case: if not pks: @@ -527,6 +530,29 @@ class TableInsertView(BaseView): result["rows"] = [dict(r) for r in fetched_rows.rows] else: result["rows"] = rows + # We track the number of rows requested, but do not attempt to show which were actually + # inserted or upserted v.s. ignored + if upsert: + await self.ds.track_event( + UpsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ) + ) + else: + await self.ds.track_event( + InsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ignore=bool(ignore), + replace=bool(replace), + ) + ) + return Response.json(result, status=200 if upsert else 201) @@ -587,6 +613,11 @@ class TableDropView(BaseView): sqlite_utils.Database(conn)[table_name].drop() await db.execute_write_fn(drop_table) + await self.ds.track_event( + DropTableEvent( + actor=request.actor, database=database_name, table=table_name + ) + ) return Response.json({"ok": True}, status=200) diff --git a/docs/conf.py b/docs/conf.py index ca0eb986..e13882b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,8 @@ extensions = [ if not os.environ.get("DISABLE_SPHINX_INLINE_TABS"): extensions += ["sphinx_inline_tabs"] +autodoc_member_order = "bysource" + extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), } diff --git a/docs/events.rst b/docs/events.rst new file mode 100644 index 00000000..f150ac02 --- /dev/null +++ b/docs/events.rst @@ -0,0 +1,14 @@ +.. _events: + +Events +====== + +Datasette includes a mechanism for tracking events that occur while the software is running. This is primarily intended to be used by plugins, which can both trigger events and listen for events. + +The core Datasette application triggers events when certain things happen. This page describes those events. + +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): + +.. automodule:: datasette.events + :members: + :exclude-members: Event diff --git a/docs/index.rst b/docs/index.rst index 66bbd5a4..ce1ed2eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,5 +63,6 @@ Contents plugin_hooks testing_plugins internals + events contributing changelog diff --git a/docs/internals.rst b/docs/internals.rst index d8f86251..bd7a70b5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -593,6 +593,26 @@ Using either of these pattern will result in the in-memory database being served This removes a database that has been previously added. ``name=`` is the unique name of that database. +.. _datasette_track_event: + +await .track_event(event) +------------------------- + +``event`` - ``Event`` + An instance of a subclass of ``datasette.events.Event``. + +Plugins can call this to track events, using classes they have previously registered. See :ref:`plugin_event_tracking` for details. + +The event will then be passed to all plugins that have registered to receive events using the :ref:`plugin_hook_track_event` hook. + +Example usage, assuming the plugin has previously registered the ``BanUserEvent`` class: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) + .. _datasette_sign: .sign(value, namespace="default") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index da69c6c9..1a88cd31 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1759,3 +1759,103 @@ top_canned_query(datasette, request, database, query_name) The name of the canned query. Returns HTML to be displayed at the top of the canned query page. + +.. _plugin_event_tracking: + +Event tracking +-------------- + +Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. + +Plugins can register to receive events using the ``track_event`` plugin hook. + +They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. + +.. _plugin_hook_track_event: + +track_event(datasette, event) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``event`` - ``Event`` + Information about the event, represented as an instance of a subclass of the ``Event`` base class. + +This hook will be called any time an event is tracked by code that calls the :ref:`datasette.track_event(...) ` internal method. + +The ``event`` object will always have the following properties: + +- ``name``: a string representing the name of the event, for example ``logout`` or ``create-table``. +- ``actor``: a dictionary representing the actor that triggered the event, or ``None`` if the event was not triggered by an actor. +- ``created``: a ``datatime.datetime`` object in the ``timezone.utc`` timezone representing the time the event object was created. + +Other properties on the event will be available depending on the type of event. You can also access those as a dictionary using ``event.properties()``. + +The events fired by Datasette core are :ref:`documented here `. + +This example plugin logs details of all events to standard error: + +.. code-block:: python + + from datasette import hookimpl + import json + import sys + + + @hookimpl + def track_event(event): + name = event.name + actor = event.actor + properties = event.properties() + msg = json.dumps( + { + "name": name, + "actor": actor, + "properties": properties, + } + ) + print(msg, file=sys.stderr, flush=True) + + +.. _plugin_hook_register_events: + +register_events(datasette) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. + +This example registers event subclasses for ``ban-user`` and ``unban-user`` events: + +.. code-block:: python + + from dataclasses import dataclass + from datasette import hookimpl, Event + + + @dataclass + class BanUserEvent(Event): + name = "ban-user" + user: dict + + + @dataclass + class UnbanUserEvent(Event): + name = "unban-user" + user: dict + + + @hookimpl + def register_events(): + return [BanUserEvent, UnbanUserEvent] + +The plugin can then call ``datasette.track_event(...)`` to send a ``ban-user`` event: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) diff --git a/docs/plugins.rst b/docs/plugins.rst index 2ec03701..1a72af95 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -228,6 +228,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "skip_csrf" ] }, + { + "name": "datasette.events", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "register_events" + ] + }, { "name": "datasette.facets", "static": false, diff --git a/tests/conftest.py b/tests/conftest.py index 31336aea..445de057 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import asyncio import httpx import os import pathlib @@ -8,7 +7,8 @@ import re import subprocess import tempfile import time -import trustme +from dataclasses import dataclass, field +from datasette import Event, hookimpl try: @@ -164,6 +164,35 @@ def check_permission_actions_are_documented(): ) +class TrackEventPlugin: + __name__ = "TrackEventPlugin" + + @dataclass + class OneEvent(Event): + name = "one" + + extra: str + + @hookimpl + def register_events(self, datasette): + async def inner(): + return [self.OneEvent] + + return inner + + @hookimpl + def track_event(self, datasette, event): + datasette._tracked_events = getattr(datasette, "_tracked_events", []) + datasette._tracked_events.append(event) + + +@pytest.fixture(scope="session", autouse=True) +def install_event_tracking_plugin(): + from datasette.plugins import pm + + pm.register(TrackEventPlugin(), name="TrackEventPlugin") + + @pytest.fixture(scope="session") def ds_localhost_http_server(): ds_proc = subprocess.Popen( diff --git a/tests/test_api.py b/tests/test_api.py index 93ca43eb..177dc95c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -786,7 +786,12 @@ async def test_threads_json(ds_client): @pytest.mark.asyncio async def test_plugins_json(ds_client): response = await ds_client.get("/-/plugins.json") - assert EXPECTED_PLUGINS == sorted(response.json(), key=lambda p: p["name"]) + # Filter out TrackEventPlugin + actual_plugins = sorted( + [p for p in response.json() if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert EXPECTED_PLUGINS == actual_plugins # Try with ?all=1 response = await ds_client.get("/-/plugins.json?all=1") names = {p["name"] for p in response.json()} diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1787e06f..9caf9fdf 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import sqlite3 +from .utils import last_event import pytest import time @@ -49,6 +50,14 @@ async def test_insert_row(ds_write): assert response.json()["rows"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace @pytest.mark.asyncio @@ -68,6 +77,16 @@ async def test_insert_rows(ds_write, return_rows): headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 20 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace + actual_rows = [ dict(r) for r in ( @@ -353,6 +372,16 @@ async def test_insert_ignore_replace( headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert event.ignore == ignore + assert event.replace == replace + actual_rows = [ dict(r) for r in ( @@ -427,6 +456,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): ) assert response.status_code == 200 assert response.json()["ok"] is True + + # Analytics event + event = last_event(ds_write) + assert event.name == "upsert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "upsert_test" + if should_return: # We only expect it to return rows corresponding to those we sent expected_returned_rows = expected_rows[: len(input["rows"])] @@ -530,6 +567,13 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 + + # Analytics event + event = last_event(ds_write) + assert event.name == "delete-row" + assert event.database == "data" + assert event.table == table + assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) @@ -610,6 +654,13 @@ async def test_update_row(ds_write, input, expected_errors, use_return): for k, v in input.items(): assert returned_row[k] == v + # Analytics event + event = last_event(ds_write) + assert event.actor == {"id": "root", "token": "dstok"} + assert event.database == "data" + assert event.table == "docs" + assert event.pks == [str(pk)] + # And fetch the row to check it's updated response = await ds_write.client.get( "/data/docs/{}.json?_shape=array".format(pk), @@ -676,6 +727,13 @@ async def test_drop_table(ds_write, scenario): headers=_headers(token), ) assert response2.json() == {"ok": True} + # Check event + event = last_event(ds_write) + assert event.name == "drop-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.table == "docs" + assert event.database == "data" + # Table should 404 assert (await ds_write.client.get("/data/docs")).status_code == 404 @@ -1096,6 +1154,12 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response + # create-table event + if expected_status == 201: + event = last_event(ds_write) + assert event.name == "create-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.schema.startswith("CREATE TABLE ") @pytest.mark.asyncio diff --git a/tests/test_auth.py b/tests/test_auth.py index 33cf9b35..f2359df7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import app_client -from .utils import cookie_was_deleted +from .utils import cookie_was_deleted, last_event from click.testing import CliRunner from datasette.utils import baseconv from datasette.cli import cli @@ -19,6 +19,10 @@ async def test_auth_token(ds_client): assert {"a": {"id": "root"}} == ds_client.ds.unsign( response.cookies["ds_actor"], "actor" ) + # Should have recorded a login event + event = last_event(ds_client.ds) + assert event.name == "login" + assert event.actor == {"id": "root"} # Check that a second with same token fails assert ds_client.ds._root_token is None assert (await ds_client.get(path)).status_code == 403 @@ -57,7 +61,7 @@ async def test_actor_cookie_that_expires(ds_client, offset, expected): cookie = ds_client.ds.sign( {"a": {"id": "test"}, "e": baseconv.base62.encode(expires_at)}, "actor" ) - response = await ds_client.get("/", cookies={"ds_actor": cookie}) + await ds_client.get("/", cookies={"ds_actor": cookie}) assert ds_client.ds._last_request.scope["actor"] == expected @@ -86,6 +90,10 @@ def test_logout(app_client): csrftoken_from=True, cookies={"ds_actor": app_client.actor_cookie({"id": "test"})}, ) + # Should have recorded a logout event + event = last_event(app_client.ds) + assert event.name == "logout" + assert event.actor == {"id": "test"} # The ds_actor cookie should have been unset assert cookie_was_deleted(response4, "ds_actor") # Should also have set a message @@ -185,6 +193,13 @@ def test_auth_create_token( for error in errors: assert '

    {}

    '.format(error) in response2.text else: + # Check create-token event + event = last_event(app_client.ds) + assert event.name == "create-token" + assert event.expires_after == expected_duration + assert isinstance(event.restrict_all, list) + assert isinstance(event.restrict_database, dict) + assert isinstance(event.restrict_resource, dict) # Extract token from page token = response2.text.split('value="dstok_')[1].split('"')[0] details = app_client.ds.unsign(token, "token") diff --git a/tests/test_cli.py b/tests/test_cli.py index 080e8353..9cc18c6e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -100,7 +100,11 @@ def test_spatialite_error_if_cannot_find_load_extension_spatialite(): def test_plugins_cli(app_client): runner = CliRunner() result1 = runner.invoke(cli, ["plugins"]) - assert json.loads(result1.output) == EXPECTED_PLUGINS + actual_plugins = sorted( + [p for p in json.loads(result1.output) if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert actual_plugins == EXPECTED_PLUGINS # Try with --all result2 = runner.invoke(cli, ["plugins", "--all"]) names = [p["name"] for p in json.loads(result2.output)] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5bfb6132..dad4f2ca 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,8 +9,9 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner +from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Permission +from datasette import cli, hookimpl, Event, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 @@ -18,6 +19,7 @@ from datasette.utils import CustomRow, StartupError from jinja2.environment import Template from jinja2 import ChoiceLoader, FileSystemLoader import base64 +import datetime import importlib import json import os @@ -1437,3 +1439,30 @@ async def test_hook_top_canned_query(ds_client): assert "Xtop_query:fixtures:from_hook:xyz" in response.text finally: pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_track_event(): + datasette = Datasette(memory=True) + from .conftest import TrackEventPlugin + + await datasette.invoke_startup() + await datasette.track_event( + TrackEventPlugin.OneEvent(actor=None, extra="extra extra") + ) + assert len(datasette._tracked_events) == 1 + assert isinstance(datasette._tracked_events[0], TrackEventPlugin.OneEvent) + event = datasette._tracked_events[0] + assert event.name == "one" + assert event.properties() == {"extra": "extra extra"} + # Should have a recent created as well + created = event.created + assert isinstance(created, datetime.datetime) + assert created.tzinfo == datetime.timezone.utc + + +@pytest.mark.asyncio +async def test_hook_register_events(): + datasette = Datasette(memory=True) + await datasette.invoke_startup() + assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) diff --git a/tests/utils.py b/tests/utils.py index 84d5b1df..9b31abde 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,11 @@ from datasette.utils.sqlite import sqlite3 +def last_event(datasette): + events = getattr(datasette, "_tracked_events", []) + return events[-1] if events else None + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) From 2e4a03b2c461ca20ff789146a006ddd126013ee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:31:26 -0800 Subject: [PATCH 1190/1866] Run coverage on Python 3.12 - #2245 I hoped this would run slightly faster than 3.9 but there doesn't appear to be a performance improvement. --- .github/workflows/test-coverage.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index bd720664..7a08e401 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,18 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install Python dependencies run: | python -m pip install --upgrade pip From bcf7ef963f6e1eb0a64b2a0bb4af0ae7a197d1d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 19:45:05 -0800 Subject: [PATCH 1191/1866] YAML/JSON examples for allow blocks --- docs/authentication.rst | 270 ++++++++++++++++++++++++++++++++++------ 1 file changed, 231 insertions(+), 39 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a301113a..8758765d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -80,13 +80,35 @@ The standard way to define permissions in Datasette is to use an ``"allow"`` blo The most basic form of allow block is this (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: root + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: root + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "root" + } } - } +.. [[[end]]] This will match any actors with an ``"id"`` property of ``"root"`` - for example, an actor that looks like this: @@ -99,29 +121,98 @@ This will match any actors with an ``"id"`` property of ``"root"`` - for example An allow block can specify "deny all" using ``false`` (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: false + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": false - } +.. tab:: YAML + + .. code-block:: yaml + + allow: false + +.. tab:: JSON + + .. code-block:: json + + { + "allow": false + } +.. [[[end]]] An ``"allow"`` of ``true`` allows all access (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": true - } +.. tab:: YAML + + .. code-block:: yaml + + allow: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": true + } +.. [[[end]]] Allow keys can provide a list of values. These will match any actor that has any of those values (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ] + } } - } +.. [[[end]]] This will match any actor with an ``"id"`` of either ``"simon"`` or ``"cleopaws"``. @@ -129,53 +220,154 @@ Actors can have properties that feature a list of values. These will be matched .. code-block:: json - { - "id": "simon", - "roles": ["staff", "developer"] - } + { + "id": "simon", + "roles": ["staff", "developer"] + } This allow block will provide access to any actor that has ``"developer"`` as one of their roles (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + roles: + - developer + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "roles": ["developer"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + roles: + - developer + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "roles": [ + "developer" + ] + } } - } +.. [[[end]]] Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on. If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to match any logged-in user specify the following (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: "*" + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: "*" + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "*" + } } - } +.. [[[end]]] You can specify that only unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + unauthenticated: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + unauthenticated: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "unauthenticated": true + } } - } +.. [[[end]]] Allow keys act as an "or" mechanism. An actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. The following block will allow users with either a ``role`` of ``"ops"`` OR users who have an ``id`` of ``"simon"`` or ``"cleopaws"``: -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + role: ops + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"], +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + role: ops + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ], "role": "ops" + } } - } +.. [[[end]]] `Demo for cleopaws `__, `demo for ops role `__, `demo for an actor matching neither rule `__. From b466749e88b2ffbd925b6b3e777c8527ebc54e78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 20:03:19 -0800 Subject: [PATCH 1192/1866] Filled out docs/configuration.rst, closes #2246 --- docs/changelog.rst | 2 +- docs/configuration.rst | 297 ++++++++++++++++++++++++++++++++++++-- docs/custom_templates.rst | 156 +------------------- docs/metadata_doc.py | 8 +- docs/plugin_hooks.rst | 2 +- docs/plugins.rst | 2 +- 6 files changed, 294 insertions(+), 173 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index af3d2a0b..04ce9583 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -610,7 +610,7 @@ JavaScript modules To use modules, JavaScript needs to be included in `` + +You can also specify a SRI (subresource integrity hash) for these assets: + +.. [[[cog + config_example(cog, """ + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_css_urls": [ + { + "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", + "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" + } + ], + "extra_js_urls": [ + { + "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", + "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" + } + ] + } +.. [[[end]]] + +This will produce: + +.. code-block:: html + + + + +Modern browsers will only execute the stylesheet or JavaScript if the SRI hash +matches the content served. You can generate hashes using `www.srihash.org `_ + +Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: + +.. [[[cog + config_example(cog, """ + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } +.. [[[end]]] + +Will produce this HTML: + +.. code-block:: html + + + + + diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index d8e4ac96..534d8b33 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -5,159 +5,6 @@ Custom pages and templates Datasette provides a number of ways of customizing the way data is displayed. -.. _customization_css_and_javascript: - -Custom CSS and JavaScript -------------------------- - -When you launch Datasette, you can specify a custom configuration file like this:: - - datasette mydb.db --config datasette.yaml - -Your ``datasette.yaml`` file can include links that look like this: - -.. [[[cog - from metadata_doc import config_example - config_example(cog, """ - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - } -.. [[[end]]] - -The extra CSS and JavaScript files will be linked in the ```` of every page: - -.. code-block:: html - - - - -You can also specify a SRI (subresource integrity hash) for these assets: - -.. [[[cog - config_example(cog, """ - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - } -.. [[[end]]] - -This will produce: - -.. code-block:: html - - - - -Modern browsers will only execute the stylesheet or JavaScript if the SRI hash -matches the content served. You can generate hashes using `www.srihash.org `_ - -Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: - -.. [[[cog - config_example(cog, """ - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_js_urls": [ - { - "url": "https://example.datasette.io/module.js", - "module": true - } - ] - } -.. [[[end]]] - -Will produce this HTML: - -.. code-block:: html - - - CSS classes on the ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -258,9 +105,10 @@ The following URLs will now serve the content from those CSS and JS files:: http://localhost:8001/assets/styles.css http://localhost:8001/assets/app.js -You can reference those files from ``datasette.yaml`` like so: +You can reference those files from ``datasette.yaml`` like this, see :ref:`custom CSS and JavaScript ` for more details: .. [[[cog + from metadata_doc import config_example config_example(cog, """ extra_css_urls: - /assets/styles.css diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index a8f13414..ad85bf52 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -24,17 +24,19 @@ def metadata_example(cog, data=None, yaml=None): cog.out("\n") -def config_example(cog, input): +def config_example( + cog, input, yaml_title="datasette.yaml", json_title="datasette.json" +): if type(input) is str: data = YAML().load(input) output_yaml = input else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out("\n.. tab:: datasette.yaml\n\n") + cog.out("\n.. tab:: {}\n\n".format(yaml_title)) cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out("\n\n.. tab:: datasette.json\n\n") + cog.out("\n\n.. tab:: {}\n\n".format(json_title)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1a88cd31..d9d135e5 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -270,7 +270,7 @@ you have one: Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. -If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. +If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`configuration_reference_css_js` for more details. .. code-block:: python diff --git a/docs/plugins.rst b/docs/plugins.rst index 1a72af95..03ddf8f0 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -328,7 +328,7 @@ To write that to a ``requirements.txt`` file, run this:: Plugin configuration -------------------- -Plugins can have their own configuration, embedded in a :ref:`configuration` file. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. +Plugins can have their own configuration, embedded in a :ref:`configuration file `. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. Here is an example of some plugin configuration for a specific table: From 4da581d09bbed2377682630c147e05d78c48f7e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:40:49 -0800 Subject: [PATCH 1193/1866] Link to config reference --- docs/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.rst b/docs/settings.rst index 1d4baf90..d1553703 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -15,6 +15,8 @@ You can set multiple settings at once like this:: --setting sql_time_limit_ms 3500 \ --setting max_returned_rows 2000 +Settings can also be specified :ref:`in the database.yaml configuration file `. + .. _config_dir: Configuration directory mode From d4bc2b2dfc728017c8f669c1714f20b89655557c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:44:16 -0800 Subject: [PATCH 1194/1866] Remove fail_if_plugins_in_metadata, part of #2248 --- datasette/app.py | 8 ++------ datasette/cli.py | 3 +-- datasette/utils/__init__.py | 15 --------------- tests/test_plugins.py | 8 -------- 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 530f79bc..0143223a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -77,7 +77,6 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, - fail_if_plugins_in_metadata, tilde_decode, to_css_class, urlsafe_components, @@ -336,16 +335,13 @@ class Datasette: ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: - metadata = fail_if_plugins_in_metadata( - parse_metadata(fp.read()), metadata_files[0].name - ) + metadata = parse_metadata(fp.read()) if config_dir and config_files and not config: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - self._metadata_local = fail_if_plugins_in_metadata(metadata or {}) - + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: # Resolve spatialite, if requested diff --git a/datasette/cli.py b/datasette/cli.py index 91f38f69..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -33,7 +33,6 @@ from .utils import ( initial_path_for_datasette, pairs_to_nested_config, temporary_docker_directory, - fail_if_plugins_in_metadata, value_as_boolean, SpatialiteNotFound, StaticMount, @@ -543,7 +542,7 @@ def serve( metadata_data = None if metadata: - metadata_data = fail_if_plugins_in_metadata(parse_metadata(metadata.read())) + metadata_data = parse_metadata(metadata.read()) config_data = None if config: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 196e1682..75f1c2f4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1272,21 +1272,6 @@ def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> return result -def fail_if_plugins_in_metadata(metadata: dict, filename=None): - """If plugin config is inside metadata, raise an Exception""" - if metadata is not None and metadata.get("plugins") is not None: - suggested_extension = ( - ".yaml" - if filename is not None - and (filename.endswith(".yaml") or filename.endswith(".yml")) - else ".json" - ) - raise Exception( - f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' - ) - return metadata - - def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm diff --git a/tests/test_plugins.py b/tests/test_plugins.py index dad4f2ca..f26e3652 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -883,14 +883,6 @@ def test_hook_forbidden(restore_working_directory): ) -def test_plugin_config_in_metadata(): - with pytest.raises( - Exception, - match="Datasette no longer accepts plugin configuration in --metadata", - ): - Datasette(memory=True, metadata={"plugins": {}}) - - @pytest.mark.asyncio async def test_hook_handle_exception(ds_client): await ds_client.get("/trigger-error?x=123") From be4f02335fb35d40a763d07b2d4e880b90083e53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:33:33 -0800 Subject: [PATCH 1195/1866] Treat plugins in metadata as if they were in config, closes #2248 --- datasette/app.py | 6 +++++ datasette/utils/__init__.py | 40 +++++++++++++++++++++++++++++ tests/test_plugins.py | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 0143223a..634283ff 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, + move_plugins, parse_metadata, resolve_env_secrets, resolve_routes, @@ -341,6 +342,11 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) + # Move any "plugins" settings from metadata to config - updates them in place + metadata = metadata or {} + config = config or {} + move_plugins(metadata, config) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 75f1c2f4..cc175b01 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1287,3 +1287,43 @@ def make_slot_function(name, datasette, request, **kwargs): return markupsafe.Markup("".join(html_bits)) return inner + + +def move_plugins(source, destination): + """ + Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. + After moving, recursively remove any keys in the source that are left empty. + """ + + def recursive_move(src, dest, path=None): + if path is None: + path = [] + for key, value in list(src.items()): + new_path = path + [key] + if key == "plugins": + # Navigate and create the hierarchy in destination if needed + d = dest + for step in path: + d = d.setdefault(step, {}) + # Move the plugins + d[key] = value + # Remove the plugins from source + src.pop(key, None) + elif isinstance(value, dict): + recursive_move(value, dest, new_path) + # After moving, check if the current dictionary is empty and remove it if so + if not value: + src.pop(key, None) + + def prune_empty_dicts(d): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + recursive_move(source, destination) + prune_empty_dicts(source) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f26e3652..a53fc118 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1458,3 +1458,54 @@ async def test_hook_register_events(): datasette = Datasette(memory=True) await datasette.invoke_startup() assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) + + +@pytest.mark.parametrize( + "metadata,config,expected_metadata,expected_config", + ( + ( + # Instance level + {"plugins": {"datasette-foo": "bar"}}, + {}, + {}, + {"plugins": {"datasette-foo": "bar"}}, + ), + ( + # Database level + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + {}, + {}, + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + ), + ( + # Table level + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + {}, + {}, + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + ), + ( + # Keep other keys + {"plugins": {"datasette-foo": "bar"}, "other": "key"}, + {"original_config": "original"}, + {"other": "key"}, + {"original_config": "original", "plugins": {"datasette-foo": "bar"}}, + ), + ), +) +def test_metadata_plugin_config_treated_as_config( + metadata, config, expected_metadata, expected_config +): + ds = Datasette(metadata=metadata, config=config) + actual_metadata = ds.metadata() + assert "plugins" not in actual_metadata + assert actual_metadata == expected_metadata + assert ds.config == expected_config From 6ccef35cc92cc2357c2b2a9aa003b7334b2459eb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:42:45 -0800 Subject: [PATCH 1196/1866] More links between events documentation --- docs/events.rst | 2 +- docs/plugin_hooks.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/events.rst b/docs/events.rst index f150ac02..b86c8025 100644 --- a/docs/events.rst +++ b/docs/events.rst @@ -7,7 +7,7 @@ Datasette includes a mechanism for tracking events that occur while the software The core Datasette application triggers events when certain things happen. This page describes those events. -Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes - or additional classes :ref:`registered by other plugins `. .. automodule:: datasette.events :members: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d9d135e5..16f5cebb 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1769,7 +1769,7 @@ Datasette includes an internal mechanism for tracking analytical events. This ca Plugins can register to receive events using the ``track_event`` plugin hook. -They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. +They can also define their own events for other plugins to receive using the :ref:`register_events() plugin hook `, combined with calls to the :ref:`datasette.track_event() internal method `. .. _plugin_hook_track_event: @@ -1826,7 +1826,7 @@ register_events(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the :ref:`datasette.track_event() ` method. This example registers event subclasses for ``ban-user`` and ``unban-user`` events: From 4ea109ac4dd17392c85ca7d5934009f9a9488a9d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:47:41 -0800 Subject: [PATCH 1197/1866] Two spaces is aesthetically more pleasing here --- docs/settings.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index d1553703..c4b4ba82 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -11,9 +11,9 @@ Datasette supports a number of settings. These can be set using the ``--setting You can set multiple settings at once like this:: datasette mydatabase.db \ - --setting default_page_size 50 \ - --setting sql_time_limit_ms 3500 \ - --setting max_returned_rows 2000 + --setting default_page_size 50 \ + --setting sql_time_limit_ms 3500 \ + --setting max_returned_rows 2000 Settings can also be specified :ref:`in the database.yaml configuration file `. @@ -25,10 +25,10 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: datasette one.db two.db \ - --metadata=metadata.json \ - --template-dir=templates/ \ - --plugins-dir=plugins \ - --static css:css + --metadata=metadata.json \ + --template-dir=templates/ \ + --plugins-dir=plugins \ + --static css:css As an alternative to this, you can run Datasette in *configuration directory* mode. Create a directory with the following structure:: From 5ea7098e4da5fe8576f6452dbfac86e6aedba397 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Feb 2024 10:15:21 -0800 Subject: [PATCH 1198/1866] Fixed an unnecessary f-string --- demos/plugins/example_js_manager_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py index 7db45464..2705f2c5 100644 --- a/demos/plugins/example_js_manager_plugins.py +++ b/demos/plugins/example_js_manager_plugins.py @@ -16,6 +16,6 @@ def extra_js_urls(view_name): if view_name in PERMITTED_VIEWS: return [ { - "url": f"/static/table-example-plugins.js", + "url": "/static/table-example-plugins.js", } ] From 7219a56d1e8b5d076037aeeec2583ad4fc3cacb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 10:34:10 -0800 Subject: [PATCH 1199/1866] 3 space indent, not 2 --- docs/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index a835ace9..79e2a1ca 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -145,8 +145,8 @@ Settings """ # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 """).strip() ) .. ]]] @@ -157,8 +157,8 @@ Settings # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 .. tab:: datasette.json From 503545b20363ac15d3664bec7e6c4522ff271668 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 11:47:17 -0800 Subject: [PATCH 1200/1866] JavaScript plugins documentation, closes #2250 --- docs/index.rst | 1 + docs/javascript_plugins.rst | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 docs/javascript_plugins.rst diff --git a/docs/index.rst b/docs/index.rst index ce1ed2eb..e3036618 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,7 @@ Contents custom_templates plugins writing_plugins + javascript_plugins plugin_hooks testing_plugins internals diff --git a/docs/javascript_plugins.rst b/docs/javascript_plugins.rst new file mode 100644 index 00000000..e7ee6817 --- /dev/null +++ b/docs/javascript_plugins.rst @@ -0,0 +1,159 @@ +.. _javascript_plugins: + +JavaScript plugins +================== + +Datasette can run custom JavaScript in several different ways: + +- Datasette plugins written in Python can use the :ref:`extra_js_urls() ` or :ref:`extra_body_script() ` plugin hooks to inject JavaScript into a page +- Datasette instances with :ref:`custom templates ` can include additional JavaScript in those templates +- The ``extra_js_urls`` key in ``datasette.yaml`` :ref:`can be used to include extra JavaScript ` + +There are no limitations on what this JavaScript can do. It is executed directly by the browser, so it can manipulate the DOM, fetch additional data and do anything else that JavaScript is capable of. + +.. warning:: + Custom JavaScript has security implications, especially for authenticated Datasette instances where the JavaScript might run in the context of the authenticated user. It's important to carefully review any JavaScript you run in your Datasette instance. + +.. _javascript_datasette_init: + +The datasette_init event +------------------------ + +Datasette emits a custom event called ``datasette_init`` when the page is loaded. This event is dispatched on the ``document`` object, and includes a ``detail`` object with a reference to the :ref:`datasetteManager ` object. + +Your JavaScript code can listen out for this event using ``document.addEventListener()`` like this: + +.. code-block:: javascript + + document.addEventListener("datasette_init", function (evt) { + const manager = evt.detail; + console.log("Datasette version:", manager.VERSION); + }); + +.. _javascript_datasette_manager: + +datasetteManager +---------------- + +The ``datasetteManager`` object + +``VERSION`` - string + The version of Datasette + +``plugins`` - ``Map()`` + A Map of currently loaded plugin names to plugin implementations + +``registerPlugin(name, implementation)`` + Call this to register a plugin, passing its name and implementation + +``selectors`` - object + An object providing named aliases to useful CSS selectors, :ref:`listed below ` + +.. _javascript_plugin_objects: + +JavaScript plugin objects +------------------------- + +JavaScript plugins are blocks of code that can be registered with Datasette using the ``registerPlugin()`` method on the :ref:`datasetteManager ` object. + +The ``implementation`` object passed to this method should include a ``version`` key defining the plugin version, and one or more of the following named functions providing the implementation of the plugin: + +.. _javascript_plugins_makeAboveTablePanelConfigs: + +makeAboveTablePanelConfigs() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method should return a JavaScript array of objects defining additional panels to be added to the top of the table page. Each object should have the following: + +``id`` - string + A unique string ID for the panel, for example ``map-panel`` +``label`` - string + A human-readable label for the panel +``render(node)`` - function + A function that will be called with a DOM node to render the panel into + +This example shows how a plugin might define a single panel: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('panel-plugin', { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: 'First panel', + render: node => { + node.innerHTML = '

    My custom panel

    This is a custom panel that I added using a JavaScript plugin

    '; + } + } + ] + } + }); + }); + +When a page with a table loads, all registered plugins that implement ``makeAboveTablePanelConfigs()`` will be called and panels they return will be added to the top of the table page. + +.. _javascript_plugins_makeColumnActions: + +makeColumnActions(columnDetails) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method, if present, will be called when Datasette is rendering the cog action menu icons that appear at the top of the table view. By default these include options like "Sort ascending/descending" and "Facet by this", but plugins can return additional actions to be included in this menu. + +The method will be called with a ``columnDetails`` object with the following keys: + +``columnName`` - string + The name of the column +``columnNotNull`` - boolean + True if the column is defined as NOT NULL +``columnType`` - string + The SQLite data type of the column +``isPk`` - boolean + True if the column is part of the primary key + +It should return a JavaScript array of objects each with a ``label`` and ``onClick`` property: + +``label`` - string + The human-readable label for the action +``onClick(evt)`` - function + A function that will be called when the action is clicked + +The ``evt`` object passed to the ``onClick`` is the standard browser event object that triggered the click. + +This example plugin adds two menu items - one to copy the column name to the clipboard and another that displays the column metadata in an ``alert()`` window: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('column-name-plugin', { + version: 0.1, + makeColumnActions: (columnDetails) => { + return [ + { + label: 'Copy column to clipboard', + onClick: async (evt) => { + await navigator.clipboard.writeText(columnDetails.columnName) + } + }, + { + label: 'Alert column metadata', + onClick: () => alert(JSON.stringify(columnDetails, null, 2)) + } + ]; + } + }); + }); + +.. _javascript_datasette_manager_selectors: + +Selectors +--------- + +These are available on the ``selectors`` property of the :ref:`javascript_datasette_manager` object. + +.. literalinclude:: ../datasette/static/datasette-manager.js + :language: javascript + :start-at: const DOM_SELECTORS = { + :end-at: }; From efc73575548b0bbaca4bdc8de40fc6939bb88428 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:01:03 -0800 Subject: [PATCH 1201/1866] Remove Using YAML for metadata section No longer necessary now we show YAML and JSON examples everywhere. --- docs/changelog.rst | 2 +- docs/metadata.rst | 29 +---------------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04ce9583..f4b928e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1190,7 +1190,7 @@ Also in this release: 0.40 (2020-04-21) ----------------- -* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. See :ref:`metadata_yaml`. (:issue:`713`) +* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. (:issue:`713`) * Removed support for ``datasette publish now``, which used the the now-retired Zeit Now v1 hosting platform. A new plugin, `datasette-publish-now `__, can be installed to publish data to Zeit (`now Vercel `__) Now v2. (:issue:`710`) * Fixed a bug where the ``extra_template_vars(request, view_name)`` plugin hook was not receiving the correct ``view_name``. (:issue:`716`) * Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`setting_template_debug`). (:issue:`693`) diff --git a/docs/metadata.rst b/docs/metadata.rst index b4dc90f9..f3ca68ac 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -53,7 +53,7 @@ Your ``metadata.yaml`` file can look something like this: .. [[[end]]] -Choosing YAML over JSON adds support for multi-line strings and comments, see :ref:`metadata_yaml`. +Choosing YAML over JSON adds support for multi-line strings and comments. The above metadata will be displayed on the index page of your Datasette-powered site. The source and license information will also be included in the footer of @@ -664,33 +664,6 @@ SpatiaLite tables are automatically hidden) using ``"hidden": true``: } .. [[[end]]] -.. _metadata_yaml: - -Using YAML for metadata ------------------------ - -Datasette accepts YAML as an alternative to JSON for your metadata configuration file. -YAML is particularly useful for including multiline HTML and SQL strings, plus inline comments. - -Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`canned_queries`. - -.. code-block:: yaml - - title: Demonstrating Metadata from YAML - description_html: |- -

    This description includes a long HTML string

    -
      -
    • YAML is better for embedding HTML strings than JSON!
      • -
    - license: ODbL - license_url: https://opendatacommons.org/licenses/odbl/ - databases: - fixtures: - tables: - no_primary_key: - hidden: true - - .. _metadata_reference: Metadata reference From 85a1dfe6e07fcdd7ec8f83cb5b3a8f023659d064 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:43:50 -0800 Subject: [PATCH 1202/1866] Configuration via the command-line section Closes #2252 Closes #2156 --- docs/configuration.rst | 78 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 79e2a1ca..425024da 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,7 +5,7 @@ Configuration Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``--config``/ ``-c`` flag: +Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``-c/--config`` flag: .. code-block:: bash @@ -13,12 +13,86 @@ Most configuration can be handled using a ``datasette.yaml`` configuration file, This file can also use JSON, as ``datasette.json``. YAML is recommended over JSON due to its support for comments and multi-line strings. +.. _configuration_cli: + +Configuration via the command-line +---------------------------------- + +The recommended way to configure Datasette is using a ``datasette.yaml`` file passed to ``-c/--config``. You can also pass individual settings to Datasette using the ``-s/--setting`` option, which can be used multiple times: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting settings.default_page_size 50 \ + --setting settings.sql_time_limit_ms 3500 + +This option takes dotted-notation for the first argument and a value for the second argument. This means you can use it to set any configuration value that would be valid in a ``datasette.yaml`` file. + +It also works for plugin configuration, for example for `datasette-cluster-map `_: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting plugins.datasette-cluster-map.latitude_column xlat \ + --setting plugins.datasette-cluster-map.longitude_column xlon + +If the value you provide is a valid JSON object or list it will be treated as nested data, allowing you to configure plugins that accept lists such as `datasette-proxy-url `_: + +.. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-proxy-url.paths '[{"path": "/proxy", "backend": "http://example.com/"}]' + +This is equivalent to a ``datasette.yaml`` file containing the following: + +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + """).strip() + ) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + +.. tab:: datasette.json + + .. code-block:: json + + { + "plugins": { + "datasette-proxy-url": { + "paths": [ + { + "path": "/proxy", + "backend": "http://example.com/" + } + ] + } + } + } +.. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference ---------------------------- -This example shows many of the valid configuration options that can exist inside ``datasette.yaml``. +The following example shows some of the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog from metadata_doc import config_example From 1e901aa690211db36f02cc1b25246d0f56cd8720 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 12:33:46 -0800 Subject: [PATCH 1203/1866] /-/config page, closes #2254 --- datasette/app.py | 14 ++++++----- datasette/utils/__init__.py | 28 +++++++++++++++++++++ datasette/views/special.py | 4 +-- docs/introspection.rst | 11 +++++++- tests/test_api.py | 50 ++++++++++++++++++++++++++----------- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 634283ff..2e20d402 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -81,6 +81,7 @@ from .utils import ( tilde_decode, to_css_class, urlsafe_components, + redact_keys, row_sql_params_pks, ) from .utils.asgi import ( @@ -1374,6 +1375,11 @@ class Datasette: output.append(script) return output + def _config(self): + return redact_keys( + self.config, ("secret", "key", "password", "token", "hash", "dsn") + ) + def _routes(self): routes = [] @@ -1433,12 +1439,8 @@ class Datasette: r"/-/settings(\.(?Pjson))?$", ) add_route( - permanent_redirect("/-/settings.json"), - r"/-/config.json", - ) - add_route( - permanent_redirect("/-/settings"), - r"/-/config", + JsonDataView.as_view(self, "config.json", lambda: self._config()), + r"/-/config(\.(?Pjson))?$", ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cc175b01..4c940645 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -17,6 +17,7 @@ import time import types import secrets import shutil +from typing import Iterable import urllib import yaml from .shutil_backport import copytree @@ -1327,3 +1328,30 @@ def move_plugins(source, destination): recursive_move(source, destination) prune_empty_dicts(source) + + +def redact_keys(original: dict, key_patterns: Iterable) -> dict: + """ + Recursively redact sensitive keys in a dictionary based on given patterns + + :param original: The original dictionary + :param key_patterns: A list of substring patterns to redact + :return: A copy of the original dictionary with sensitive values redacted + """ + + def redact(data): + if isinstance(data, dict): + return { + k: ( + redact(v) + if not any(pattern in k for pattern in key_patterns) + else "***" + ) + for k, v in data.items() + } + elif isinstance(data, list): + return [redact(item) for item in data] + else: + return data + + return redact(original) diff --git a/datasette/views/special.py b/datasette/views/special.py index 4088a1f9..296652d0 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -42,7 +42,7 @@ class JsonDataView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response( - json.dumps(data), + json.dumps(data, default=repr), content_type="application/json; charset=utf-8", headers=headers, ) @@ -53,7 +53,7 @@ class JsonDataView(BaseView): request=request, context={ "filename": self.filename, - "data_json": json.dumps(data, indent=4), + "data_json": json.dumps(data, indent=4, default=repr), }, ) diff --git a/docs/introspection.rst b/docs/introspection.rst index e08ca911..b62197ea 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -87,7 +87,7 @@ Shows a list of currently installed plugins and their versions. `Plugins example Add ``?all=1`` to include details of the default plugins baked into Datasette. -.. _JsonDataView_config: +.. _JsonDataView_settings: /-/settings ----------- @@ -105,6 +105,15 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. + +Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. + .. _JsonDataView_databases: /-/databases diff --git a/tests/test_api.py b/tests/test_api.py index 177dc95c..0a1f3725 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -846,20 +846,6 @@ async def test_settings_json(ds_client): } -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path,expected_redirect", - ( - ("/-/config.json", "/-/settings.json"), - ("/-/config", "/-/settings"), - ), -) -async def test_config_redirects_to_settings(ds_client, path, expected_redirect): - response = await ds_client.get(path) - assert response.status_code == 301 - assert response.headers["Location"] == expected_redirect - - test_json_columns_default_expected = [ {"intval": 1, "strval": "s", "floatval": 0.5, "jsonval": '{"foo": "bar"}'} ] @@ -1039,3 +1025,39 @@ async def test_tilde_encoded_database_names(db_name): # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config,expected", + ( + ({}, {}), + ({"plugins": {"datasette-foo": "bar"}}, {"plugins": {"datasette-foo": "bar"}}), + # Test redaction + ( + { + "plugins": { + "datasette-auth": {"secret_key": "key"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "password"}, + "datasette-sentry": { + "dsn": "sentry:///foo", + }, + } + }, + { + "plugins": { + "datasette-auth": {"secret_key": "***"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "***"}, + "datasette-sentry": {"dsn": "***"}, + } + }, + ), + ), +) +async def test_config_json(config, expected): + "/-/config.json should return redacted configuration" + ds = Datasette(config=config) + response = await ds.client.get("/-/config.json") + assert response.json() == expected From 5a63ecc5577d070f28bf5daa34aaaf3dadfd2e4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:03:19 -0800 Subject: [PATCH 1204/1866] Rename metadata= to table_config= in facet code, refs #2247 --- datasette/facets.py | 41 ++++++++++++++++++++-------------------- datasette/views/table.py | 2 +- tests/test_facets.py | 4 ++-- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b23615fe..f1cfc68f 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -11,8 +11,8 @@ from datasette.utils import ( ) -def load_facet_configs(request, table_metadata): - # Given a request and the metadata configuration for a table, return +def load_facet_configs(request, table_config): + # Given a request and the configuration for a table, return # a dictionary of selected facets, their lists of configs and for each # config whether it came from the request or the metadata. # @@ -20,21 +20,21 @@ def load_facet_configs(request, table_metadata): # {"source": "metadata", "config": config1}, # {"source": "request", "config": config2}]} facet_configs = {} - table_metadata = table_metadata or {} - metadata_facets = table_metadata.get("facets", []) - for metadata_config in metadata_facets: - if isinstance(metadata_config, str): + table_config = table_config or {} + table_facet_configs = table_config.get("facets", []) + for facet_config in table_facet_configs: + if isinstance(facet_config, str): type = "column" - metadata_config = {"simple": metadata_config} + facet_config = {"simple": facet_config} else: assert ( - len(metadata_config.values()) == 1 + len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, metadata_config = list(metadata_config.items())[0] - if isinstance(metadata_config, str): - metadata_config = {"simple": metadata_config} + type, facet_config = list(facet_config.items())[0] + if isinstance(facet_config, str): + facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( - {"source": "metadata", "config": metadata_config} + {"source": "metadata", "config": facet_config} ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): @@ -45,13 +45,12 @@ def load_facet_configs(request, table_metadata): elif key.startswith("_facet_"): type = key[len("_facet_") :] for value in values: - # The value is the config - either JSON or not - if value.startswith("{"): - config = json.loads(value) - else: - config = {"simple": value} + # The value is the facet_config - either JSON or not + facet_config = ( + json.loads(value) if value.startswith("{") else {"simple": value} + ) facet_configs.setdefault(type, []).append( - {"source": "request", "config": config} + {"source": "request", "config": facet_config} ) return facet_configs @@ -75,7 +74,7 @@ class Facet: sql=None, table=None, params=None, - metadata=None, + table_config=None, row_count=None, ): assert table or sql, "Must provide either table= or sql=" @@ -86,12 +85,12 @@ class Facet: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] - self.metadata = metadata + self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: self.row_count = row_count def get_configs(self): - configs = load_facet_configs(self.request, self.metadata) + configs = load_facet_configs(self.request, self.table_config) return configs.get(self.type) or [] def get_querystring_pairs(self): diff --git a/datasette/views/table.py b/datasette/views/table.py index 3b812c01..22722847 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1275,7 +1275,7 @@ async def table_view_data( sql=sql_no_order_no_limit, params=params, table=table_name, - metadata=table_metadata, + table_config=table_metadata, row_count=extra_count, ) ) diff --git a/tests/test_facets.py b/tests/test_facets.py index 85c8f85b..76344108 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -82,7 +82,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) suggestions = [s["name"] for s in await facet.suggest()] assert [ @@ -278,7 +278,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) buckets, timed_out = await facet.facet_results() assert [] == timed_out From 5d21057cf1e4171bc2741d3d8d9da83aee1165b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:22:03 -0800 Subject: [PATCH 1205/1866] /-/config example, refs #2254 --- docs/introspection.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/introspection.rst b/docs/introspection.rst index b62197ea..ff78ec78 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -110,7 +110,17 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. +Shows the :ref:`configuration ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. `Config example `_: + +.. code-block:: json + + { + "settings": { + "template_debug": true, + "trace_debug": true, + "force_https_urls": true + } + } Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. From 69c6e953231078ab18ba0807e5fe2a4e20e84093 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:27:20 -0800 Subject: [PATCH 1206/1866] Fixed a bunch of unused imports spotted with ruff --- datasette/cli.py | 1 - datasette/forbidden.py | 1 - datasette/handle_exception.py | 4 +--- datasette/permissions.py | 2 +- datasette/url_builder.py | 2 +- datasette/views/base.py | 1 - datasette/views/index.py | 3 --- ruff.toml | 1 + tests/conftest.py | 2 +- tests/test_black.py | 2 -- tests/test_cli.py | 2 -- tests/test_cli_serve_get.py | 2 +- tests/test_config_dir.py | 2 -- tests/test_internals_datasette.py | 1 - tests/test_plugins.py | 7 ++----- tests/test_table_html.py | 2 +- 16 files changed, 9 insertions(+), 26 deletions(-) create mode 100644 ruff.toml diff --git a/datasette/cli.py b/datasette/cli.py index 1a5a8af3..0c8a8541 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -15,7 +15,6 @@ import sys import textwrap import webbrowser from .app import ( - OBSOLETE_SETTINGS, Datasette, DEFAULT_SETTINGS, SETTINGS, diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 156a44d4..41c48396 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,4 +1,3 @@ -from os import stat from datasette import hookimpl, Response diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index bef6b4ee..1a0ac979 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,14 +1,12 @@ from datasette import hookimpl, Response -from .utils import await_me_maybe, add_cors_headers +from .utils import add_cors_headers from .utils.asgi import ( Base400, - Forbidden, ) from .views.base import DatasetteError from markupsafe import Markup import pdb import traceback -from .plugins import pm try: import rich diff --git a/datasette/permissions.py b/datasette/permissions.py index 152f1721..bd42158e 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, fields +from dataclasses import dataclass from typing import Optional diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 574bf3c1..9c6bbde0 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib diff --git a/datasette/views/base.py b/datasette/views/base.py index bdc1e9cf..9d7a854c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -10,7 +10,6 @@ from markupsafe import escape import pint -from datasette import __version__ from datasette.database import QueryInterrupted from datasette.utils.asgi import Request from datasette.utils import ( diff --git a/datasette/views/index.py b/datasette/views/index.py index 595cf234..2cb18b1c 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,12 +1,9 @@ import json -from datasette.plugins import pm from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ -from markupsafe import Markup - from .base import BaseView diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..0deb884c --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 160 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 445de057..168194d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import re import subprocess import tempfile import time -from dataclasses import dataclass, field +from dataclasses import dataclass from datasette import Event, hookimpl diff --git a/tests/test_black.py b/tests/test_black.py index d09b2514..ccf51171 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1,8 +1,6 @@ import black from click.testing import CliRunner from pathlib import Path -import pytest -import sys code_root = Path(__file__).parent.parent diff --git a/tests/test_cli.py b/tests/test_cli.py index 9cc18c6e..bda17eed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,6 @@ from .fixtures import ( TestClient as _TestClient, EXPECTED_PLUGINS, ) -import asyncio from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve @@ -19,7 +18,6 @@ import pytest import sys import textwrap from unittest import mock -import urllib def test_inspect_cli(app_client): diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ff2429c6..1088d906 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,4 +1,4 @@ -from datasette.cli import cli, serve +from datasette.cli import cli from datasette.plugins import pm from click.testing import CliRunner import textwrap diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 748412c3..66114a27 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -3,11 +3,9 @@ import pathlib import pytest from datasette.app import Datasette -from datasette.cli import cli from datasette.utils.sqlite import sqlite3 from datasette.utils import StartupError from .fixtures import TestClient as _TestClient -from click.testing import CliRunner PLUGIN = """ from datasette import hookimpl diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index c30bb748..2614e02e 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -7,7 +7,6 @@ from datasette import Forbidden, Context from datasette.app import Datasette, Database from itsdangerous import BadSignature import pytest -from typing import Optional @pytest.fixture diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a53fc118..c02c94cc 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,6 +1,5 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( - app_client, app_client, make_app_client, TABLES, @@ -9,14 +8,12 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner -from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Event, Permission +from datasette import cli, hookimpl, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 -from datasette.utils import CustomRow, StartupError -from jinja2.environment import Template +from datasette.utils import StartupError from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 0604d34c..2a658663 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1,4 +1,4 @@ -from datasette.app import Datasette, Database +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, From f0491038523e000b97a18d2e9a23faee62208083 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:33:18 -0800 Subject: [PATCH 1207/1866] datasette.table_metadata() is now await datasette.table_config(), refs #2247 --- datasette/app.py | 2 +- datasette/database.py | 2 +- datasette/filters.py | 2 +- datasette/views/row.py | 2 +- datasette/views/table.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 2e20d402..b0b8f041 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1202,7 +1202,7 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - def table_metadata(self, database, table): + async def table_config(self, database, table): """Fetch table-specific metadata.""" return ( (self.metadata("databases") or {}) diff --git a/datasette/database.py b/datasette/database.py index f2c980d7..1c1b3e1b 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -418,7 +418,7 @@ class Database: return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): - explicit_label_column = self.ds.table_metadata(self.name, table).get( + explicit_label_column = (await self.ds.table_config(self.name, table)).get( "label_column" ) if explicit_label_column: diff --git a/datasette/filters.py b/datasette/filters.py index 73eea857..4d9580d8 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -50,7 +50,7 @@ def search_filters(request, database, table, datasette): extra_context = {} # Figure out which fts_table to use - table_metadata = datasette.table_metadata(database, table) + table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) fts_table = request.args.get("_fts_table") fts_table = fts_table or table_metadata.get("fts_table") diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b646641..7b43b893 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -89,7 +89,7 @@ class RowView(DataView): "columns": columns, "primary_keys": resolved.pks, "primary_key_values": pk_values, - "units": self.ds.table_metadata(database, table).get("units", {}), + "units": (await self.ds.table_config(database, table)).get("units", {}), } if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): diff --git a/datasette/views/table.py b/datasette/views/table.py index 22722847..2b5b7c24 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,7 +142,7 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) column_descriptions = table_metadata.get("columns") or {} column_details = { col.name: col for col in await db.table_column_details(table_name) @@ -663,7 +663,7 @@ async def _columns_to_select(table_columns, pks, request): async def _sortable_columns_for_table(datasette, database_name, table_name, use_rowid): db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: @@ -962,7 +962,7 @@ async def table_view_data( nocount = True nofacet = True - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) units = table_metadata.get("units", {}) # Arguments that start with _ and don't contain a __ are From 52a1dac5d2bac7e106c8d6ce8e0c6f1dc0141a7e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:00:55 -0800 Subject: [PATCH 1208/1866] Test proving $env works for datasette.yml, closes #2255 --- tests/test_plugins.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c02c94cc..40d01c71 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -231,10 +231,18 @@ async def test_plugin_config(ds_client): @pytest.mark.asyncio -async def test_plugin_config_env(ds_client): - os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" - assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - del os.environ["FOO_ENV"] +async def test_plugin_config_env(ds_client, monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT") + assert ds_client.ds.plugin_config("env-plugin") == {"foo": "FROM_ENVIRONMENT"} + + +@pytest.mark.asyncio +async def test_plugin_config_env_from_config(monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT_2") + datasette = Datasette( + config={"plugins": {"env-plugin": {"setting": {"$env": "FOO_ENV"}}}} + ) + assert datasette.plugin_config("env-plugin") == {"setting": "FROM_ENVIRONMENT_2"} @pytest.mark.asyncio From 60c6692f6802dfae6a433f648f287be30ef52325 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:57:09 -0800 Subject: [PATCH 1209/1866] table_config instead of table_metadata (#2257) Table configuration that was incorrectly placed in metadata is now treated as if it was in config. New await datasette.table_config() method. Closes #2247 --- datasette/app.py | 12 +++-- datasette/database.py | 10 ++-- datasette/utils/__init__.py | 69 +++++++++++++++++++----- datasette/views/table.py | 11 ++-- tests/test_api.py | 101 +++++++++++++++++++++++++++++++++++- tests/test_html.py | 2 +- 6 files changed, 175 insertions(+), 30 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b0b8f041..373b3e95 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -75,6 +75,7 @@ from .utils import ( format_bytes, module_from_path, move_plugins, + move_table_config, parse_metadata, resolve_env_secrets, resolve_routes, @@ -346,7 +347,9 @@ class Datasette: # Move any "plugins" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - move_plugins(metadata, config) + metadata, config = move_plugins(metadata, config) + # Now migrate any known table configuration settings over as well + metadata, config = move_table_config(metadata, config) self._metadata_local = metadata or {} self.sqlite_extensions = [] @@ -1202,10 +1205,11 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - async def table_config(self, database, table): - """Fetch table-specific metadata.""" + async def table_config(self, database: str, table: str) -> dict: + """Return dictionary of configuration for specified table""" return ( - (self.metadata("databases") or {}) + (self.config or {}) + .get("databases", {}) .get(database, {}) .get("tables", {}) .get(table, {}) diff --git a/datasette/database.py b/datasette/database.py index 1c1b3e1b..fba81496 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -487,13 +487,11 @@ class Database: ) ).rows ] - # Add any from metadata.json - db_metadata = self.ds.metadata(database=self.name) - if "tables" in db_metadata: + # Add any tables marked as hidden in config + db_config = self.ds.config.get("databases", {}).get(self.name, {}) + if "tables" in db_config: hidden_tables += [ - t - for t in db_metadata["tables"] - if db_metadata["tables"][t].get("hidden") + t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] # Also mark as hidden any tables which start with the name of a hidden table # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 4c940645..fcaebe3f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -2,6 +2,7 @@ import asyncio from contextlib import contextmanager import click from collections import OrderedDict, namedtuple, Counter +import copy import base64 import hashlib import inspect @@ -17,7 +18,7 @@ import time import types import secrets import shutil -from typing import Iterable +from typing import Iterable, Tuple import urllib import yaml from .shutil_backport import copytree @@ -1290,11 +1291,24 @@ def make_slot_function(name, datasette, request, **kwargs): return inner -def move_plugins(source, destination): +def prune_empty_dicts(d: dict): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + +def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys in the source that are left empty. """ + source = copy.deepcopy(source) + destination = copy.deepcopy(destination) def recursive_move(src, dest, path=None): if path is None: @@ -1316,18 +1330,49 @@ def move_plugins(source, destination): if not value: src.pop(key, None) - def prune_empty_dicts(d): - """ - Recursively prune all empty dictionaries from a given dictionary. - """ - for key, value in list(d.items()): - if isinstance(value, dict): - prune_empty_dicts(value) - if value == {}: - d.pop(key, None) - recursive_move(source, destination) prune_empty_dicts(source) + return source, destination + + +_table_config_keys = ( + "hidden", + "sort", + "sort_desc", + "size", + "sortable_columns", + "label_column", + "facets", + "fts_table", + "fts_pk", + "searchmode", + "units", +) + + +def move_table_config(metadata: dict, config: dict): + """ + Move all known table configuration keys from metadata to config. + """ + if "databases" not in metadata: + return metadata, config + metadata = copy.deepcopy(metadata) + config = copy.deepcopy(config) + for database_name, database in metadata["databases"].items(): + if "tables" not in database: + continue + for table_name, table in database["tables"].items(): + for key in _table_config_keys: + if key in table: + config.setdefault("databases", {}).setdefault( + database_name, {} + ).setdefault("tables", {}).setdefault(table_name, {})[ + key + ] = table.pop( + key + ) + prune_empty_dicts(metadata) + return metadata, config def redact_keys(original: dict, key_patterns: Iterable) -> dict: diff --git a/datasette/views/table.py b/datasette/views/table.py index 2b5b7c24..50d2b3c2 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,11 +142,11 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = await datasette.table_config(database_name, table_name) - column_descriptions = table_metadata.get("columns") or {} + column_descriptions = datasette.metadata("columns", database_name, table_name) or {} column_details = { col.name: col for col in await db.table_column_details(table_name) } + table_config = await datasette.table_config(database_name, table_name) pks = await db.primary_keys(table_name) pks_for_display = pks if not pks_for_display: @@ -193,7 +193,6 @@ async def display_columns_and_rows( "raw": pk_path, "value": markupsafe.Markup( '{flat_pks}'.format( - base_url=base_url, table_path=datasette.urls.table(database_name, table_name), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), @@ -274,9 +273,9 @@ async def display_columns_and_rows( ), ) ) - elif column in table_metadata.get("units", {}) and value != "": + elif column in table_config.get("units", {}) and value != "": # Interpret units using pint - value = value * ureg(table_metadata["units"][column]) + value = value * ureg(table_config["units"][column]) # Pint uses floating point which sometimes introduces errors in the compact # representation, which we have to round off to avoid ugliness. In the vast # majority of cases this rounding will be inconsequential. I hope. @@ -591,7 +590,7 @@ class TableDropView(BaseView): try: data = json.loads(await request.post_body()) confirm = data.get("confirm") - except json.JSONDecodeError as e: + except json.JSONDecodeError: pass if not confirm: diff --git a/tests/test_api.py b/tests/test_api.py index 0a1f3725..8cb73dbb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -771,7 +771,7 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_metadata_json(ds_client): response = await ds_client.get("/-/metadata.json") - assert response.json() == METADATA + assert response.json() == ds_client.ds.metadata() @pytest.mark.asyncio @@ -1061,3 +1061,102 @@ async def test_config_json(config, expected): ds = Datasette(config=config) response = await ds.client.get("/-/config.json") assert response.json() == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metadata,expected_config,expected_metadata", + ( + ({}, {}, {}), + ( + # Metadata input + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ], + }, + "no_primary_key": {"sortable_columns": [], "hidden": True}, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + "primary_key_multiple_columns_explicit_label": { + "label_column": "content2" + }, + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + }, + } + }, + }, + # Should produce a config with just the table configuration keys + { + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ] + }, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + # These one get redacted: + "no_primary_key": "***", + "primary_key_multiple_columns_explicit_label": "***", + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + } + } + } + }, + # And metadata with everything else + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + } + } + }, + }, + ), + ), +) +async def test_upgrade_metadata(metadata, expected_config, expected_metadata): + ds = Datasette(metadata=metadata) + response = await ds.client.get("/-/config.json") + assert response.json() == expected_config + response2 = await ds.client.get("/-/metadata.json") + assert response2.json() == expected_metadata diff --git a/tests/test_html.py b/tests/test_html.py index 86895844..8229b166 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -753,7 +753,7 @@ async def test_metadata_json_html(ds_client): response = await ds_client.get("/-/metadata") assert response.status_code == 200 pre = Soup(response.content, "html.parser").find("pre") - assert METADATA == json.loads(pre.text) + assert ds_client.ds.metadata() == json.loads(pre.text) @pytest.mark.asyncio From 9ac9f0152f1d3396d01ceddfcaf07fd1cf3f7168 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:18:38 -0800 Subject: [PATCH 1210/1866] Migrate allow from metadata to config if necessary, closes #2249 --- datasette/app.py | 6 +++--- datasette/utils/__init__.py | 9 +++++---- tests/test_permissions.py | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 373b3e95..af8cfeab 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,7 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, - move_plugins, + move_plugins_and_allow, move_table_config, parse_metadata, resolve_env_secrets, @@ -344,10 +344,10 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - # Move any "plugins" settings from metadata to config - updates them in place + # Move any "plugins" and "allow" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - metadata, config = move_plugins(metadata, config) + metadata, config = move_plugins_and_allow(metadata, config) # Now migrate any known table configuration settings over as well metadata, config = move_table_config(metadata, config) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index fcaebe3f..f2cd7eb0 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1302,10 +1302,11 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ - Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. - After moving, recursively remove any keys in the source that are left empty. + Move 'plugins' and 'allow' keys from source to destination dictionary. Creates + hierarchy in destination if needed. After moving, recursively remove any keys + in the source that are left empty. """ source = copy.deepcopy(source) destination = copy.deepcopy(destination) @@ -1315,7 +1316,7 @@ def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: path = [] for key, value in list(src.items()): new_path = path + [key] - if key == "plugins": + if key in ("plugins", "allow"): # Navigate and create the hierarchy in destination if needed d = dest for step in path: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 9917b749..6713b850 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -89,10 +89,11 @@ def test_view_padlock(allow, expected_anon, expected_auth, path, padlock_client) ({"id": "root"}, 403, 200), ], ) -def test_view_database(allow, expected_anon, expected_auth): - with make_app_client( - config={"databases": {"fixtures": {"allow": allow}}} - ) as client: +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_database(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = {key: {"databases": {"fixtures": {"allow": allow}}}} + with make_app_client(**kwargs) as client: for path in ( "/fixtures", "/fixtures/compound_three_primary_keys", @@ -173,16 +174,19 @@ def test_database_list_respects_view_table(): ({"id": "root"}, 403, 200), ], ) -def test_view_table(allow, expected_anon, expected_auth): - with make_app_client( - config={ +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_table(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = { + key: { "databases": { "fixtures": { "tables": {"compound_three_primary_keys": {"allow": allow}} } } } - ) as client: + } + with make_app_client(**kwargs) as client: anon_response = client.get("/fixtures/compound_three_primary_keys") assert expected_anon == anon_response.status if allow and anon_response.status == 200: From ad01f9d3217f2447b0a321ee7731900dac7e3e6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:24:24 -0800 Subject: [PATCH 1211/1866] 1.0a8 release notes Closes #2243 * Changelog for jinja2_environment_from_request and plugin_hook_slots * track_event() in changelog * Remove Using YAML for metadata section - no longer necessary now we show YAML and JSON examples everywhere. * Configuration via the command-line section - #2252 * JavaScript plugins in release notes, refs #2052 * /-/config in changelog, refs #2254 Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254 --- docs/changelog.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4b928e3..1f47b429 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,83 @@ Changelog ========= +.. _v1_0_a8: + +1.0a8 (2024-02-01) +------------------ + +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. + +Configuration +~~~~~~~~~~~~~ + +- Plugin configuration now lives in the :ref:`datasette.yaml configuration file `, passed to Datasette using the ``-c/--config`` option. Thanks, Alex Garcia. (:issue:`2093`) + + .. code-block:: bash + + datasette -c datasette.yaml + + Where ``datasette.yaml`` contains configuration that looks like this: + + .. code-block:: yaml + + plugins: + datasette-cluster-map: + latitude_column: xlat + longitude_column: xlon + + Previously plugins were configured in ``metadata.yaml``, which was confusing as plugin settings were unrelated to database and table metadata. +- The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`) + + The above YAML configuration example using ``-s/--setting`` looks like this: + + .. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-cluster-map.latitude_column xlat \ + -s plugins.datasette-cluster-map.longitude_column xlon + +- The new ``/-/config`` page shows the current instance configuration, after redacting keys that could contain sensitive data such as API keys or passwords. (:issue:`2254`) + +- Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) + +JavaScript plugins +~~~~~~~~~~~~~~~~~~ + +Datasette now includes a :ref:`JavaScript plugins mechanism `, allowing JavaScript to customize Datasette in a way that can collaborate with other plugins. + +This provides two initial hooks, with more to come in the future: + +- :ref:`makeAboveTablePanelConfigs() ` can add additional panels to the top of the table page. +- :ref:`makeColumnActions() ` can add additional actions to the column menu. + +Thanks `Cameron Yick `__ for contributing this feature. (`#2052 `__) + +Plugin hooks +~~~~~~~~~~~~ + +- New :ref:`plugin_hook_jinja2_environment_from_request` plugin hook, which can be used to customize the current Jinja environment based on the incoming request. This can be used to modify the template lookup path based on the incoming request hostname, among other things. (:issue:`2225`) +- New :ref:`family of template slot plugin hooks `: ``top_homepage``, ``top_database``, ``top_table``, ``top_row``, ``top_query``, ``top_canned_query``. Plugins can use these to provide additional HTML to be injected at the top of the corresponding pages. (:issue:`1191`) +- New :ref:`track_event() mechanism ` for plugins to emit and receive events when certain events occur within Datasette. (:issue:`2240`) + - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. + - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. + - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. +- New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) + +Documentation +~~~~~~~~~~~~~ + +- Documentation describing :ref:`how to write tests that use signed actor cookies ` using ``datasette.client.actor_cookie()``. (:issue:`1830`) +- Documentation on how to :ref:`register a plugin for the duration of a test `. (:issue:`2234`) +- The :ref:`configuration documentation ` now shows examples of both YAML and JSON for each setting. + +Minor fixes +~~~~~~~~~~~ + +- Datasette no longer attempts to run SQL queries in parallel when rendering a table page, as this was leading to some rare crashing bugs. (:issue:`2189`) +- Fixed warning: ``DeprecationWarning: pkg_resources is deprecated as an API`` (:issue:`2057`) +- Fixed bug where ``?_extra=columns`` parameter returned an incorrectly shaped response. (:issue:`2230`) + .. _v0_64_6: 0.64.6 (2023-12-22) From c64453a4a137ea815f4d94a99cdc4c7709734839 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:28:22 -0800 Subject: [PATCH 1212/1866] Fix the date on the 1.0a8 release (due to go tomorrow) Refs #2258 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f47b429..5e9e3ba2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a8: -1.0a8 (2024-02-01) +1.0a8 (2024-02-07) ------------------ This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. From d0089ba7769d58b97c5dc1e07969246502d97544 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:30:30 -0800 Subject: [PATCH 1213/1866] Note in changelog about datasette publish, refs #2195 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5e9e3ba2..e17dc2f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,8 @@ Configuration - Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) +Note that the ``datasette publish`` command has not yet been updated to accept a ``datasette.yaml`` configuration file. This will be addressed in :issue:`2195` but for the moment you can include those settings in ``metadata.yaml`` instead. + JavaScript plugins ~~~~~~~~~~~~~~~~~~ From df8d1c055a48a36693d9caec3915eb93c1acefc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:59:58 -0800 Subject: [PATCH 1214/1866] Mention JS plugins in release intro --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e17dc2f8..a73635a8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 1.0a8 (2024-02-07) ------------------ -This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. Configuration ~~~~~~~~~~~~~ From 1e31821d9ff2bc81c62bcd442214e9098cf785a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:25:47 -0800 Subject: [PATCH 1215/1866] Link to events docs from changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a73635a8..dfcf492f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,7 @@ Plugin hooks - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. + - Datasette core now emits ``login``, ``logout``, ``create-token``, ``create-table``, ``drop-table``, ``insert-rows``, ``upsert-rows``, ``update-row``, ``delete-row`` events, :ref:`documented here `. - New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) Documentation From e0794ddd52697812848c0b59f68e49a2e9361693 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:32:47 -0800 Subject: [PATCH 1216/1866] Link to annotated release notes blog post --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index dfcf492f..d164f71d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,8 @@ Changelog This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. +See `Datasette 1.0a8: JavaScript plugins, new plugin hooks and plugin configuration in datasette.yaml `__ for an annotated version of these release notes. + Configuration ~~~~~~~~~~~~~ From 9989f257094daaf26e0cb0cebe31f17f19d4cad2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:34:05 -0800 Subject: [PATCH 1217/1866] Release 1.0a8 Refs Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254, #2258 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 75d44727..e43b9918 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8.dev1" +__version__ = "1.0a8" __version_info__ = tuple(__version__.split(".")) From 569aacd39bcc5529b2f463c89c616e3ada21c560 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 22:53:14 -0800 Subject: [PATCH 1218/1866] Link to /en/latest/ changelog --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57f17a5c..662f2a11 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) From 900d15bcb81c90d26cfebc3fe463c4b0465832c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:21:13 -0800 Subject: [PATCH 1219/1866] alter table support for /db/-/create API, refs #2101 --- datasette/default_permissions.py | 10 ++++- datasette/events.py | 25 +++++++++++ datasette/views/database.py | 61 +++++++++++++++++++++++--- docs/authentication.rst | 12 ++++++ tests/test_api_write.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 8 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index d29dbe84..c13f2ed2 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -8,7 +8,6 @@ from typing import Union, Tuple @hookimpl def register_permissions(): return ( - # name, abbr, description, takes_database, takes_resource, default Permission( name="view-instance", abbr="vi", @@ -109,6 +108,14 @@ def register_permissions(): takes_resource=False, default=False, ), + Permission( + name="alter-table", + abbr="at", + description="Alter tables", + takes_database=True, + takes_resource=True, + default=False, + ), Permission( name="drop-table", abbr="dt", @@ -129,6 +136,7 @@ def permission_allowed_default(datasette, actor, action, resource): "debug-menu", "insert-row", "create-table", + "alter-table", "drop-table", "delete-row", "update-row", diff --git a/datasette/events.py b/datasette/events.py index 96244779..ae90972d 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -108,6 +108,30 @@ class DropTableEvent(Event): table: str +@dataclass +class AlterTableEvent(Event): + """ + Event name: ``alter-table`` + + A table has been altered. + + :ivar database: The name of the database where the table was altered + :type database: str + :ivar table: The name of the table that was altered + :type table: str + :ivar before_schema: The table's SQL schema before the alteration + :type before_schema: str + :ivar after_schema: The table's SQL schema after the alteration + :type after_schema: str + """ + + name = "alter-table" + database: str + table: str + before_schema: str + after_schema: str + + @dataclass class InsertRowsEvent(Event): """ @@ -203,6 +227,7 @@ def register_events(): LogoutEvent, CreateTableEvent, CreateTokenEvent, + AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/views/database.py b/datasette/views/database.py index 6d17b16c..bd55064f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -792,7 +792,17 @@ class MagicParameters(dict): class TableCreateView(BaseView): name = "table-create" - _valid_keys = {"table", "rows", "row", "columns", "pk", "pks", "ignore", "replace"} + _valid_keys = { + "table", + "rows", + "row", + "columns", + "pk", + "pks", + "ignore", + "replace", + "alter", + } _supported_column_types = { "text", "integer", @@ -876,6 +886,20 @@ class TableCreateView(BaseView): ): return _error(["Permission denied - need insert-row"], 403) + alter = False + if rows or row: + if not table_exists: + # if table is being created for the first time, alter=True + alter = True + else: + # alter=True only if they request it AND they have permission + if data.get("alter"): + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=database_name + ): + return _error(["Permission denied - need alter-table"], 403) + alter = True + if columns: if rows or row: return _error(["Cannot specify columns with rows or row"]) @@ -939,10 +963,18 @@ class TableCreateView(BaseView): return _error(["pk cannot be changed for existing table"]) pks = actual_pks + initial_schema = None + if table_exists: + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + def create_table(conn): table = sqlite_utils.Database(conn)[table_name] if rows: - table.insert_all(rows, pk=pks or pk, ignore=ignore, replace=replace) + table.insert_all( + rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter + ) else: table.create( {c["name"]: c["type"] for c in columns}, @@ -954,6 +986,18 @@ class TableCreateView(BaseView): schema = await db.execute_write_fn(create_table) except Exception as e: return _error([str(e)]) + + if initial_schema is not None and initial_schema != schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=schema, + ) + ) + table_url = self.ds.absolute_url( request, self.ds.urls.table(db.name, table_name) ) @@ -970,11 +1014,14 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) - await self.ds.track_event( - CreateTableEvent( - request.actor, database=db.name, table=table_name, schema=schema + + if not table_exists: + # Only log creation if we created a table + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) ) - ) return Response.json(details, status=201) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8758765d..87ee6385 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1217,6 +1217,18 @@ Actor is allowed to create a database table. Default *deny*. +.. _permissions_alter_table: + +alter-table +----------- + +Actor is allowed to alter a database table. + +``resource`` - tuple: (string, string) + The name of the database, then the name of the table + +Default *deny*. + .. _permissions_drop_table: drop-table diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9caf9fdf..30cbfbab 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1349,3 +1349,77 @@ async def test_method_not_allowed(ds_write, path): "ok": False, "error": "Method not allowed", } + + +@pytest.mark.asyncio +async def test_create_uses_alter_by_default_for_new_table(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ] + * 100 + + [ + {"name": "Row 2", "extra": "Extra"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + event = last_event(ds_write) + assert event.name == "create-table" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +async def test_create_using_alter_against_existing_table( + ds_write, has_alter_permission +): + token = write_token( + ds_write, permissions=["ir", "ct"] + (["at"] if has_alter_permission else []) + ) + # First create the table + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + # Now try to insert more rows using /-/create with alter=True + response2 = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [{"name": "Row 2", "extra": "extra"}], + "pk": "id", + "alter": True, + }, + headers=_headers(token), + ) + if not has_alter_permission: + assert response2.status_code == 403 + assert response2.json() == { + "ok": False, + "errors": ["Permission denied - need alter-table"], + } + else: + assert response2.status_code == 201 + # It should have altered the table + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema From 574687834f4bd8e73281731b8ff01bfe093fecb5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:33:41 -0800 Subject: [PATCH 1220/1866] Docs for /db/-/create alter: true option, refs #2101 --- docs/json_api.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index 16b997eb..68a0c984 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -834,19 +834,22 @@ To create a table, make a ``POST`` to ``//-/create``. This requires th The JSON here describes the table that will be created: -* ``table`` is the name of the table to create. This field is required. -* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. +* ``table`` is the name of the table to create. This field is required. +* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. - - ``name`` is the name of the column. This is required. - - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``name`` is the name of the column. This is required. + - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. -* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. +* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. - If the primary key is an integer column, it will be configured to automatically increment for each new record. + If the primary key is an integer column, it will be configured to automatically increment for each new record. - If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an integer ID column for you. + If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an auto-incrementing integer ID column for you. -* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: @@ -925,6 +928,8 @@ You can avoid this error by passing the same ``"ignore": true`` or ``"replace": To use the ``"replace": true`` option you will also need the :ref:`permissions_update_row` permission. +Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`permissions_alter_table` permission. + .. _TableDropView: Dropping tables From b5ccc4d60844a24fdf91c3f317d8cda4a285a58d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:35:12 -0800 Subject: [PATCH 1221/1866] Test for Permission denied - need alter-table --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 30cbfbab..abf9a88a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1377,7 +1377,7 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): @pytest.mark.asyncio -@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +@pytest.mark.parametrize("has_alter_permission", (True, False)) async def test_create_using_alter_against_existing_table( ds_write, has_alter_permission ): From 528d89d1a3d6ff85047a7eef9a7623efdd2fb19f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:14:12 -0800 Subject: [PATCH 1222/1866] alter: true support for /-/insert and /-/upsert, refs #2101 --- datasette/views/table.py | 48 +++++++++++++++++++++++++++++++++++----- docs/json_api.rst | 6 ++++- tests/test_api_write.py | 48 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50d2b3c2..fcbe253d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,7 +8,12 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted -from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent +from datasette.events import ( + AlterTableEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, +) from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -388,7 +393,7 @@ class TableInsertView(BaseView): extras = { key: value for key, value in data.items() if key not in ("row", "rows") } - valid_extras = {"return", "ignore", "replace"} + valid_extras = {"return", "ignore", "replace", "alter"} invalid_extras = extras.keys() - valid_extras if invalid_extras: return _errors( @@ -397,7 +402,6 @@ class TableInsertView(BaseView): if extras.get("ignore") and extras.get("replace"): return _errors(['Cannot use "ignore" and "replace" at the same time']) - # Validate columns of each row columns = set(await db.table_columns(table_name)) columns.update(pks_list) @@ -412,7 +416,7 @@ class TableInsertView(BaseView): ) ) invalid_columns = set(row.keys()) - columns - if invalid_columns: + if invalid_columns and not extras.get("alter"): errors.append( "Row {} has invalid columns: {}".format( i, ", ".join(sorted(invalid_columns)) @@ -476,10 +480,23 @@ class TableInsertView(BaseView): ignore = extras.get("ignore") replace = extras.get("replace") + alter = extras.get("alter") if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + initial_schema = None + if alter: + # Must have alter-table permission + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(database_name, table_name) + ): + return _error(["Permission denied for alter-table"], 403) + # Track initial schema to check if it changed later + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + should_return = bool(extras.get("return", False)) row_pk_values_for_later = [] if should_return and upsert: @@ -489,9 +506,13 @@ class TableInsertView(BaseView): table = sqlite_utils.Database(conn)[table_name] kwargs = {} if upsert: - kwargs["pk"] = pks[0] if len(pks) == 1 else pks + kwargs = { + "pk": pks[0] if len(pks) == 1 else pks, + "alter": alter, + } else: - kwargs = {"ignore": ignore, "replace": replace} + # Insert + kwargs = {"ignore": ignore, "replace": replace, "alter": alter} if should_return and not upsert: rowids = [] method = table.upsert if upsert else table.insert @@ -552,6 +573,21 @@ class TableInsertView(BaseView): ) ) + if initial_schema is not None: + after_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + if initial_schema != after_schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=after_schema, + ) + ) + return Response.json(result, status=200 if upsert else 201) diff --git a/docs/json_api.rst b/docs/json_api.rst index 68a0c984..000f532d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -618,6 +618,8 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _TableUpsertView: Upserting rows @@ -728,6 +730,8 @@ When using upsert you must provide the primary key column (or columns if the tab If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowUpdateView: Updating a row @@ -849,7 +853,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. -* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index abf9a88a..9e1d73e0 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -60,6 +60,27 @@ async def test_insert_row(ds_write): assert not event.replace +@pytest.mark.asyncio +async def test_insert_row_alter(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={ + "row": {"title": "Test", "score": 1.2, "age": 5, "extra": "extra"}, + "alter": True, + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert response.json()["ok"] is True + assert response.json()["rows"][0]["extra"] == "extra" + # Analytics event + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema + + @pytest.mark.asyncio @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): @@ -278,16 +299,27 @@ async def test_insert_rows(ds_write, return_rows): 403, ["Permission denied: need both insert-row and update-row"], ), + # Alter table forbidden without alter permission + ( + "/data/docs/-/upsert", + {"rows": [{"id": 1, "title": "One", "extra": "extra"}], "alter": True}, + "update-and-insert-but-no-alter", + 403, + ["Permission denied for alter-table"], + ), ), ) async def test_insert_or_upsert_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): - token = write_token(ds_write) + token_permissions = [] if special_case == "insert-but-not-update": - token = write_token(ds_write, permissions=["ir", "vi"]) + token_permissions = ["ir", "vi"] if special_case == "update-but-not-insert": - token = write_token(ds_write, permissions=["ur", "vi"]) + token_permissions = ["ur", "vi"] + if special_case == "update-and-insert-but-no-alter": + token_permissions = ["ur", "ir"] + token = write_token(ds_write, permissions=token_permissions) if special_case == "duplicate_id": await ds_write.get_database("data").execute_write( "insert into docs (id) values (1)" @@ -309,7 +341,9 @@ async def test_insert_or_upsert_row_errors( actor_response = ( await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) ).json() - print(actor_response) + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -434,6 +468,12 @@ async def test_insert_ignore_replace( {"id": 1, "title": "Two", "score": 1}, ], ), + ( + # Upsert with an alter + {"rows": [{"id": 1, "title": "One"}], "pk": "id"}, + {"rows": [{"id": 1, "title": "Two", "extra": "extra"}], "alter": True}, + [{"id": 1, "title": "Two", "extra": "extra"}], + ), ), ) @pytest.mark.parametrize("should_return", (False, True)) From 4e944c29e4a208f173f15ac2df6253ff90f6466f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:19:47 -0800 Subject: [PATCH 1223/1866] Corrected path used in test_update_row_check_permission --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9e1d73e0..b43ee5a6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -633,7 +633,7 @@ async def test_update_row_check_permission(ds_write, scenario): pk = await _insert_row(ds_write) - path = "/data/{}/{}/-/delete".format( + path = "/data/{}/{}/-/update".format( "docs" if scenario != "bad_table" else "bad_table", pk ) From c954795f9af9007e7c04d9b472bfd2faef647a87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:30:48 -0800 Subject: [PATCH 1224/1866] alter: true for row/-/update, refs #2101 --- datasette/views/row.py | 12 +++++++++++- docs/json_api.rst | 2 ++ tests/test_api_write.py | 43 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b43b893..4d20e41a 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -237,11 +237,21 @@ class RowUpdateView(BaseView): if not "update" in data or not isinstance(data["update"], dict): return _error(["JSON must contain an update dictionary"]) + invalid_keys = set(data.keys()) - {"update", "return", "alter"} + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + update = data["update"] + alter = data.get("alter") + if alter and not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(resolved.db.name, resolved.table) + ): + return _error(["Permission denied for alter-table"], 403) + def update_row(conn): sqlite_utils.Database(conn)[resolved.table].update( - resolved.pk_values, update + resolved.pk_values, update, alter=alter ) try: diff --git a/docs/json_api.rst b/docs/json_api.rst index 000f532d..c401d97e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -787,6 +787,8 @@ The returned JSON will look like this: Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowDeleteView: Deleting a row diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b43ee5a6..7cc38674 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -622,12 +622,17 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table")) +@pytest.mark.parametrize( + "scenario", ("no_token", "no_perm", "bad_table", "cannot_alter") +) async def test_update_row_check_permission(ds_write, scenario): if scenario == "no_token": token = "bad_token" elif scenario == "no_perm": token = write_token(ds_write, actor_id="not-root") + elif scenario == "cannot_alter": + # update-row but no alter-table: + token = write_token(ds_write, permissions=["ur"]) else: token = write_token(ds_write) @@ -637,9 +642,13 @@ async def test_update_row_check_permission(ds_write, scenario): "docs" if scenario != "bad_table" else "bad_table", pk ) + json_body = {"update": {"title": "New title"}} + if scenario == "cannot_alter": + json_body["alter"] = True + response = await ds_write.client.post( path, - json={"update": {"title": "New title"}}, + json=json_body, headers=_headers(token), ) assert response.status_code == 403 if scenario in ("no_token", "bad_token") else 404 @@ -651,6 +660,36 @@ async def test_update_row_check_permission(ds_write, scenario): ) +@pytest.mark.asyncio +async def test_update_row_invalid_key(ds_write): + token = write_token(ds_write) + + pk = await _insert_row(ds_write) + + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title"}, "bad_key": 1}, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} + + +@pytest.mark.asyncio +async def test_update_row_alter(ds_write): + token = write_token(ds_write, permissions=["ur", "at"]) + pk = await _insert_row(ds_write) + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == {"ok": True} + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", From c62cfa6de836667834b5b9a7fef2b861307ac998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:32:36 -0800 Subject: [PATCH 1225/1866] Fix upsert test to detect new alter-table event --- tests/test_api_write.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 7cc38674..2d127e1a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -499,10 +499,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): # Analytics event event = last_event(ds_write) - assert event.name == "upsert-rows" - assert event.num_rows == 1 assert event.database == "data" assert event.table == "upsert_test" + if input.get("alter"): + assert event.name == "alter-table" + assert "extra" in event.after_schema + else: + assert event.name == "upsert-rows" + assert event.num_rows == 1 if should_return: # We only expect it to return rows corresponding to those we sent From dcd9ea3622520c99a1f921766dc36ca4c0e3b796 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 14:14:58 -0800 Subject: [PATCH 1226/1866] datasette-events-db as an example of track_events() --- docs/plugin_hooks.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 16f5cebb..960dc9b6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1817,6 +1817,7 @@ This example plugin logs details of all events to standard error: ) print(msg, file=sys.stderr, flush=True) +Example: `datasette-events-db `_ .. _plugin_hook_register_events: From bd9ed62e5d8821f9dc9e035b195452980c900b3c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 18:58:12 -0800 Subject: [PATCH 1227/1866] Make ds.pemrission_allawed(..., default=) a keyword-only argument, refs #2262 --- datasette/app.py | 2 +- datasette/views/table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index af8cfeab..d943b97b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -896,7 +896,7 @@ class Datasette: await await_me_maybe(hook) async def permission_allowed( - self, actor, action, resource=None, default=DEFAULT_NOT_SET + self, actor, action, resource=None, *, default=DEFAULT_NOT_SET ): """Check permissions using the permissions_allowed plugin hook""" result = None diff --git a/datasette/views/table.py b/datasette/views/table.py index fcbe253d..1c187692 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -444,10 +444,10 @@ class TableInsertView(BaseView): # Must have insert-row AND upsert-row permissions if not ( await self.ds.permission_allowed( - request.actor, "insert-row", database_name, table_name + request.actor, "insert-row", resource=(database_name, table_name) ) and await self.ds.permission_allowed( - request.actor, "update-row", database_name, table_name + request.actor, "update-row", resource=(database_name, table_name) ) ): return _error( From 398a92cf1e54f868ff80f01634d6a814d1c61998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 20:12:22 -0800 Subject: [PATCH 1228/1866] Include database in name of _execute_writes thread, closes #2265 --- datasette/database.py | 3 +++ tests/test_api.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index fba81496..becb552c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -196,6 +196,9 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() diff --git a/tests/test_api.py b/tests/test_api.py index 8cb73dbb..7a25b55e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -780,7 +780,11 @@ async def test_threads_json(ds_client): expected_keys = {"threads", "num_threads"} if sys.version_info >= (3, 7, 0): expected_keys.update({"tasks", "num_tasks"}) - assert set(response.json().keys()) == expected_keys + data = response.json() + assert set(data.keys()) == expected_keys + # Should be at least one _execute_writes thread for __INTERNAL__ + thread_names = [thread["name"] for thread in data["threads"]] + assert "_execute_writes for database __INTERNAL__" in thread_names @pytest.mark.asyncio From 5d7997418664bcdfdba714c16bd5a67c241e8740 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Feb 2024 07:19:47 -0800 Subject: [PATCH 1229/1866] Call them "notable events" --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 960dc9b6..5372ea5e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1765,7 +1765,7 @@ Returns HTML to be displayed at the top of the canned query page. Event tracking -------------- -Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. +Datasette includes an internal mechanism for tracking notable events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. Plugins can register to receive events using the ``track_event`` plugin hook. From b89cac3b6a63929325c067d0cf2d5748e4bf4d2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Feb 2024 18:23:54 -0800 Subject: [PATCH 1230/1866] Use MD5 usedforsecurity=False on Python 3.9 and higher to pass FIPS Closes #2270 --- datasette/database.py | 4 ++-- datasette/utils/__init__.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index becb552c..707d8f85 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,7 +1,6 @@ import asyncio from collections import namedtuple from pathlib import Path -import hashlib import janus import queue import sys @@ -15,6 +14,7 @@ from .utils import ( detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, + md5_not_usedforsecurity, sqlite_timelimit, sqlite3, table_columns, @@ -74,7 +74,7 @@ class Database: def color(self): if self.hash: return self.hash[:6] - return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): if self.path: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index f2cd7eb0..e3637f7a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -713,7 +713,7 @@ def to_css_class(s): """ if css_class_re.match(s): return s - md5_suffix = hashlib.md5(s.encode("utf8")).hexdigest()[:6] + md5_suffix = md5_not_usedforsecurity(s)[:6] # Strip leading _, - s = s.lstrip("_").lstrip("-") # Replace any whitespace with hyphens @@ -1401,3 +1401,11 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict: return data return redact(original) + + +def md5_not_usedforsecurity(s): + try: + return hashlib.md5(s.encode("utf8"), usedforsecurity=False).hexdigest() + except TypeError: + # For Python 3.8 which does not support usedforsecurity=False + return hashlib.md5(s.encode("utf8")).hexdigest() From 97de4d6362ce5a6c1e3520ecdc73b305ab269910 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 15 Feb 2024 21:35:49 -0800 Subject: [PATCH 1231/1866] Use transaction in delete_everything(), closes #2273 --- datasette/utils/internal_db.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 2e5ac53b..dd0d3a9d 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,18 +69,20 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + with conn: + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) From 47e29e948b26e8c003a03b4fc46cb635134a3958 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 10:05:18 -0800 Subject: [PATCH 1232/1866] Better comments in permission_allowed_default() --- datasette/default_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index c13f2ed2..757b3a46 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,7 +144,7 @@ def permission_allowed_default(datasette, actor, action, resource): if actor and actor.get("id") == "root": return True - # Resolve metadata view permissions + # Resolve view permissions in allow blocks in configuration if action in ( "view-instance", "view-database", @@ -158,7 +158,7 @@ def permission_allowed_default(datasette, actor, action, resource): if result is not None: return result - # Check custom permissions: blocks + # Resolve custom permissions: blocks in configuration result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) From 232a30459babebece653795d136fb6516444ecf0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 12:56:39 -0800 Subject: [PATCH 1233/1866] DATASETTE_TRACE_PLUGINS setting, closes #2274 --- datasette/plugins.py | 24 ++++++++++++++++++++++++ docs/writing_plugins.rst | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/datasette/plugins.py b/datasette/plugins.py index f7a1905f..3769a209 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,6 +1,7 @@ import importlib import os import pluggy +from pprint import pprint import sys from . import hookspecs @@ -33,6 +34,29 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) +DATASETTE_TRACE_PLUGINS = os.environ.get("DATASETTE_TRACE_PLUGINS", None) + + +def before(hook_name, hook_impls, kwargs): + print(file=sys.stderr) + print(f"{hook_name}:", file=sys.stderr) + pprint(kwargs, width=40, indent=4, stream=sys.stderr) + print("Hook implementations:", file=sys.stderr) + pprint(hook_impls, width=40, indent=4, stream=sys.stderr) + + +def after(outcome, hook_name, hook_impls, kwargs): + results = outcome.get_result() + if not isinstance(results, list): + results = [results] + print(f"Results:", file=sys.stderr) + pprint(results, width=40, indent=4, stream=sys.stderr) + + +if DATASETTE_TRACE_PLUGINS: + pm.add_hookcall_monitoring(before, after) + + DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 5c8bc4c6..2bc6bd24 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -7,6 +7,30 @@ You can write one-off plugins that apply to just one Datasette instance, or you Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. +.. _writing_plugins_tracing: + +Tracing plugin hooks +-------------------- + +The ``DATASETTE_TRACE_PLUGINS`` environment variable turns on detailed tracing showing exactly which hooks are being run. This can be useful for understanding how Datasette is using your plugin. + +.. code-block:: bash + + DATASETTE_TRACE_PLUGINS=1 datasette mydb.db + +Example output:: + + actor_from_request: + { 'datasette': , + 'request': } + Hook implementations: + [ >, + >, + >] + Results: + [{'id': 'root'}] + + .. _writing_plugins_one_off: Writing one-off plugins From 8bfa3a51c222d653f45fb48ebcb6957a85f9ea6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:29:39 -0800 Subject: [PATCH 1234/1866] Consider every plugins opinion in datasette.permission_allowed() Closes #2275, refs #2262 --- datasette/app.py | 14 +++++++++++++- docs/authentication.rst | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index d943b97b..8591af6a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -903,6 +903,8 @@ class Datasette: # Use default from registered permission, if available if default is DEFAULT_NOT_SET and action in self.permissions: default = self.permissions[action].default + opinions = [] + # Every plugin is consulted for their opinion for check in pm.hook.permission_allowed( datasette=self, actor=actor, @@ -911,9 +913,19 @@ class Datasette: ): check = await await_me_maybe(check) if check is not None: - result = check + opinions.append(check) + + result = None + # If any plugin said False it's false - the veto rule + if any(not r for r in opinions): + result = False + elif any(r for r in opinions): + # Otherwise, if any plugin said True it's true + result = True + used_default = False if result is None: + # No plugin expressed an opinion, so use the default result = default used_default = True self._permission_checks.append( diff --git a/docs/authentication.rst b/docs/authentication.rst index 87ee6385..a8dc5637 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -71,6 +71,23 @@ Datasette's built-in view permissions (``view-database``, ``view-table`` etc) de Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. +.. _authentication_permissions_explained: + +How permissions are resolved +---------------------------- + +The :ref:`datasette.permission_allowed(actor, action, resource=None, default=...)` method is called to check if an actor is allowed to perform a specific action. + +This method asks every plugin that implements the :ref:`plugin_hook_permission_allowed` hook if the actor is allowed to perform the action. + +Each plugin can return ``True`` to indicate that the actor is allowed to perform the action, ``False`` if they are not allowed and ``None`` if the plugin has no opinion on the matter. + +``False`` acts as a veto - if any plugin returns ``False`` then the permission check is denied. Otherwise, if any plugin returns ``True`` then the permission check is allowed. + +The ``resource`` argument can be used to specify a specific resource that the action is being performed against. Some permissions, such as ``view-instance``, do not involve a resource. Others such as ``view-database`` have a resource that is a string naming the database. Permissions that take both a database name and the name of a table, view or canned query within that database use a resource that is a tuple of two strings, ``(database_name, resource_name)``. + +Plugins that implement the ``permission_allowed()`` hook can decide if they are going to consider the provided resource or not. + .. _authentication_permissions_allow: Defining permissions with "allow" blocks From 244f3ff83aac19e96fab85a95ddde349079a9827 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:39:57 -0800 Subject: [PATCH 1235/1866] Test demonstrating fix for permisisons bug in #2262 --- tests/test_api_write.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2d127e1a..2aea699b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -365,6 +365,41 @@ async def test_insert_or_upsert_row_errors( assert before_count == after_count +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed", (True, False)) +async def test_upsert_permissions_per_table(ds_write, allowed): + # https://github.com/simonw/datasette/issues/2262 + token = "dstok_{}".format( + ds_write.sign( + { + "a": "root", + "token": "dstok", + "t": int(time.time()), + "_r": { + "r": { + "data": { + "docs" if allowed else "other": ["ir", "ur"], + } + } + }, + }, + namespace="token", + ) + ) + response = await ds_write.client.post( + "/data/docs/-/upsert", + json={"rows": [{"id": 1, "title": "One"}]}, + headers={ + "Authorization": "Bearer {}".format(token), + }, + ) + if allowed: + assert response.status_code == 200 + assert response.json()["ok"] is True + else: + assert response.status_code == 403 + + @pytest.mark.asyncio @pytest.mark.parametrize( "ignore,replace,expected_rows", From 3a999a85fb431594ccee1adf38721de03de19500 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:58:33 -0800 Subject: [PATCH 1236/1866] Fire insert-rows on /db/-/create if rows were inserted, refs #2260 --- datasette/views/database.py | 13 ++++++- tests/test_api_write.py | 71 +++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index bd55064f..2a8b40cc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import AlterTableEvent, CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -1022,6 +1022,17 @@ class TableCreateView(BaseView): request.actor, database=db.name, table=table_name, schema=schema ) ) + if rows: + await self.ds.track_event( + InsertRowsEvent( + request.actor, + database=db.name, + table=table_name, + num_rows=len(rows), + ignore=ignore, + replace=replace, + ) + ) return Response.json(details, status=201) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2aea699b..0eb915ba 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -857,13 +857,14 @@ async def test_drop_table(ds_write, scenario): @pytest.mark.asyncio @pytest.mark.parametrize( - "input,expected_status,expected_response", + "input,expected_status,expected_response,expected_events", ( # Permission error with a bad token ( {"table": "bad", "row": {"id": 1}}, 403, {"ok": False, "errors": ["Permission denied"]}, + [], ), # Successful creation with columns: ( @@ -910,6 +911,7 @@ async def test_drop_table(ds_write, scenario): ")" ), }, + ["create-table"], ), # Successful creation with rows: ( @@ -945,6 +947,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 2, }, + ["create-table", "insert-rows"], ), # Successful creation with row: ( @@ -973,6 +976,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create with row and no primary key ( @@ -992,6 +996,7 @@ async def test_drop_table(ds_write, scenario): "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create table with compound primary key ( @@ -1013,6 +1018,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Error: Table is required ( @@ -1024,6 +1030,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Table is required"], }, + [], ), # Error: Invalid table name ( @@ -1036,6 +1043,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Invalid table name"], }, + [], ), # Error: JSON must be an object ( @@ -1045,6 +1053,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["JSON must be an object"], }, + [], ), # Error: Cannot specify columns with rows or row ( @@ -1058,6 +1067,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify columns with rows or row"], }, + [], ), # Error: columns, rows or row is required ( @@ -1069,6 +1079,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns, rows or row is required"], }, + [], ), # Error: columns must be a list ( @@ -1081,6 +1092,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list"], }, + [], ), # Error: columns must be a list of objects ( @@ -1093,6 +1105,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list of objects"], }, + [], ), # Error: Column name is required ( @@ -1105,6 +1118,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Column name is required"], }, + [], ), # Error: Unsupported column type ( @@ -1117,6 +1131,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Unsupported column type: bad"], }, + [], ), # Error: Duplicate column name ( @@ -1132,6 +1147,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Duplicate column name: id"], }, + [], ), # Error: rows must be a list ( @@ -1144,6 +1160,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list"], }, + [], ), # Error: rows must be a list of objects ( @@ -1156,6 +1173,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list of objects"], }, + [], ), # Error: pk must be a string ( @@ -1169,6 +1187,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pk must be a string"], }, + [], ), # Error: Cannot specify both pk and pks ( @@ -1183,6 +1202,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify both pk and pks"], }, + [], ), # Error: pks must be a list ( @@ -1196,12 +1216,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pks must be a list"], }, + [], ), # Error: pks must be a list of strings ( {"table": "bad", "row": {"id": 1, "name": "Row 1"}, "pks": [1, 2]}, 400, {"ok": False, "errors": ["pks must be a list of strings"]}, + [], ), # Error: ignore and replace are mutually exclusive ( @@ -1217,6 +1239,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace are mutually exclusive"], }, + [], ), # ignore and replace require row or rows ( @@ -1230,6 +1253,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require row or rows"], }, + [], ), # ignore and replace require pk or pks ( @@ -1243,6 +1267,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ( { @@ -1255,10 +1280,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ), ) -async def test_create_table(ds_write, input, expected_status, expected_response): +async def test_create_table( + ds_write, input, expected_status, expected_response, expected_events +): + ds_write._tracked_events = [] # Special case for expected status of 403 if expected_status == 403: token = "bad_token" @@ -1272,12 +1301,9 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response - # create-table event - if expected_status == 201: - event = last_event(ds_write) - assert event.name == "create-table" - assert event.actor == {"id": "root", "token": "dstok"} - assert event.schema.startswith("CREATE TABLE ") + # Should have tracked the expected events + events = ds_write._tracked_events + assert [e.name for e in events] == expected_events @pytest.mark.asyncio @@ -1376,6 +1402,8 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) ) assert first_response.status_code == 201 + ds_write._tracked_events = [] + # Try a second time second_response = await ds_write.client.post( "/data/-/create", @@ -1387,6 +1415,10 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) rows = await ds_write.client.get("/data/test_insert_replace.json?_shape=array") assert rows.json() == expected_rows_after + # Check it fired the right events + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["insert-rows"] + @pytest.mark.asyncio async def test_create_table_error_if_pk_changed(ds_write): @@ -1471,6 +1503,7 @@ async def test_method_not_allowed(ds_write, path): @pytest.mark.asyncio async def test_create_uses_alter_by_default_for_new_table(ds_write): + ds_write._tracked_events = [] token = write_token(ds_write) response = await ds_write.client.post( "/data/-/create", @@ -1490,8 +1523,8 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): headers=_headers(token), ) assert response.status_code == 201 - event = last_event(ds_write) - assert event.name == "create-table" + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["create-table", "insert-rows"] @pytest.mark.asyncio @@ -1517,6 +1550,8 @@ async def test_create_using_alter_against_existing_table( headers=_headers(token), ) assert response.status_code == 201 + + ds_write._tracked_events = [] # Now try to insert more rows using /-/create with alter=True response2 = await ds_write.client.post( "/data/-/create", @@ -1536,8 +1571,16 @@ async def test_create_using_alter_against_existing_table( } else: assert response2.status_code == 201 + + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["alter-table", "insert-rows"] + # It should have altered the table - event = last_event(ds_write) - assert event.name == "alter-table" - assert "extra" not in event.before_schema - assert "extra" in event.after_schema + alter_event = ds_write._tracked_events[0] + assert alter_event.name == "alter-table" + assert "extra" not in alter_event.before_schema + assert "extra" in alter_event.after_schema + + insert_rows_event = ds_write._tracked_events[1] + assert insert_rows_event.name == "insert-rows" + assert insert_rows_event.num_rows == 1 From 9906f937d92c79dcc457cb057d7222ed70aef0e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:32:47 -0800 Subject: [PATCH 1237/1866] Release 1.0a9 Refs #2101, #2260, #2262, #2265, #2270, #2273, #2274, #2275 Closes #2276 --- datasette/version.py | 2 +- docs/changelog.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e43b9918..f5e07ac8 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8" +__version__ = "1.0a9" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d164f71d..e567f422 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,51 @@ Changelog ========= +.. _v1_0_a9: + +1.0a9 (2024-02-16) +------------------ + +This alpha release adds basic alter table support to the Datasette Write API and fixes a permissions bug relating to the ``/upsert`` API endpoint. + +Alter table support for create, insert, upsert and update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`JSON write API ` can now be used to apply simple alter table schema changes, provided the acting actor has the new :ref:`permissions_alter_table` permission. (:issue:`2101`) + +The only alter operation supported so far is adding new columns to an existing table. + +* The :ref:`/db/-/create ` API now adds new columns during large operations to create a table based on incoming example ``"rows"``, in the case where one of the later rows includes columns that were not present in the earlier batches. This requires the ``create-table`` but not the ``alter-table`` permission. +* When ``/db/-/create`` is called with rows in a situation where the table may have been already created, an ``"alter": true`` key can be included to indicate that any missing columns from the new rows should be added to the table. This requires the ``alter-table`` permission. +* :ref:`/db/table/-/insert ` and :ref:`/db/table/-/upsert ` and :ref:`/db/table/row-pks/-/update ` all now also accept ``"alter": true``, depending on the ``alter-table`` permission. + +Operations that alter a table now fire the new :ref:`alter-table event `. + +Permissions fix for the upsert API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`/database/table/-/upsert API ` had a minor permissions bug, only affecting Datasette instances that had configured the ``insert-row`` and ``update-row`` permissions to apply to a specific table rather than the database or instance as a whole. Full details in issue :issue:`2262`. + +To avoid similar mistakes in the future the :ref:`datasette.permission_allowed() ` method now specifies ``default=`` as a keyword-only argument. + +Permission checks now consider opinions from every plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`datasette.permission_allowed() ` method previously consulted every plugin that implemented the :ref:`permission_allowed() ` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`) + +Datasette now consults every plugin and checks to see if any of them returned ``False`` (the veto rule), and if none of them did, it then checks to see if any of them returned ``True``. + +This is explained at length in the new documentation covering :ref:`authentication_permissions_explained`. + +Other changes +~~~~~~~~~~~~~ + +- The new :ref:`DATASETTE_TRACE_PLUGINS=1 environment variable ` turns on detailed trace output for every executed plugin hook, useful for debugging and understanding how the plugin system works at a low level. (:issue:`2274`) +- Datasette on Python 3.9 or above marks its non-cryptographic uses of the MD5 hash function as ``usedforsecurity=False``, for compatibility with FIPS systems. (:issue:`2270`) +- SQL relating to :ref:`internals_internal` now executes inside a transaction, avoiding a potential database locked error. (:issue:`2273`) +- The ``/-/threads`` debug page now identifies the database in the name associated with each dedicated write thread. (:issue:`2265`) +- The ``/db/-/create`` API now fires a ``insert-rows`` event if rows were inserted after the table was created. (:issue:`2260`) + .. _v1_0_a8: 1.0a8 (2024-02-07) From e1c80efff8f4b0a53619546bb03e6dfd6cb42a32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:43:36 -0800 Subject: [PATCH 1238/1866] Note about activating alpha documentation versions on ReadTheDocs --- docs/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ef022a4d..b678e637 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -254,6 +254,7 @@ Datasette releases are performed using tags. When a new release is published on * Re-point the "latest" tag on Docker Hub to the new image * Build a wheel bundle of the underlying Python source code * Push that new wheel up to PyPI: https://pypi.org/project/datasette/ +* If the release is an alpha, navigate to https://readthedocs.org/projects/datasette/versions/ and search for the tag name in the "Activate a version" filter, then mark that version as "active" to ensure it will appear on the public ReadTheDocs documentation site. To deploy new releases you will need to have push access to the main Datasette GitHub repository. From 5e0e440f2c8a0771b761b02801456e55e95e2a04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:28:15 -0800 Subject: [PATCH 1239/1866] database.execute_write_fn(transaction=True) parameter, closes #2277 --- datasette/database.py | 31 +++++++++++++++++++++++-------- docs/internals.rst | 15 ++++++++++----- tests/test_internals_database.py | 27 +++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 707d8f85..d34aac73 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -179,17 +179,25 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def execute_write_fn(self, fn, block=True): + async def execute_write_fn(self, fn, block=True, transaction=True): if self.ds.executor is None: # non-threaded mode if self._write_connection is None: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) - return fn(self._write_connection) + if transaction: + with self._write_connection: + return fn(self._write_connection) + else: + return fn(self._write_connection) else: - return await self._send_to_write_thread(fn, block) + return await self._send_to_write_thread( + fn, block=block, transaction=transaction + ) - async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): + async def _send_to_write_thread( + self, fn, block=True, isolated_connection=False, transaction=True + ): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -202,7 +210,9 @@ class Database: self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) + self._write_queue.put( + WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + ) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -244,7 +254,11 @@ class Database: pass else: try: - result = task.fn(conn) + if task.transaction: + with conn: + result = task.fn(conn) + else: + result = task.fn(conn) except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() @@ -554,13 +568,14 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") - def __init__(self, fn, task_id, reply_queue, isolated_connection): + def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue self.isolated_connection = isolated_connection + self.transaction = transaction class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index bd7a70b5..6ca62423 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1010,7 +1010,9 @@ You can pass additional SQL parameters as a tuple or dictionary. The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. -If you pass ``block=False`` this behaviour changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. +If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. + +Each call to ``execute_write()`` will be executed inside a transaction. .. _database_execute_write_script: @@ -1019,6 +1021,8 @@ await db.execute_write_script(sql, block=True) Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() `__ method. +Each call to ``execute_write_script()`` will be executed inside a transaction. + .. _database_execute_write_many: await db.execute_write_many(sql, params_seq, block=True) @@ -1033,10 +1037,12 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() 5") - conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1069,7 +1074,7 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. -If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. +By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors. If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index dd68a6cb..57e75046 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -66,6 +66,33 @@ async def test_execute_fn(db): assert 2 == await db.execute_fn(get_1_plus_1) +@pytest.mark.asyncio +async def test_execute_fn_transaction_false(): + datasette = Datasette(memory=True) + db = datasette.add_memory_database("test_execute_fn_transaction_false") + + def run(conn): + try: + with conn: + conn.execute("create table foo (id integer primary key)") + conn.execute("insert into foo (id) values (44)") + # Table should exist + assert ( + conn.execute( + 'select count(*) from sqlite_master where name = "foo"' + ).fetchone()[0] + == 1 + ) + assert conn.execute("select id from foo").fetchall()[0][0] == 44 + raise ValueError("Cancel commit") + except ValueError: + pass + # Row should NOT exist + assert conn.execute("select count(*) from foo").fetchone()[0] == 0 + + await db.execute_write_fn(run, transaction=False) + + @pytest.mark.parametrize( "tables,exists", ( From 10f9ba1a0050724ba47a089861606bef58a4087f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:51:19 -0800 Subject: [PATCH 1240/1866] Take advantage of execute_write_fn(transaction=True) A bunch of places no longer need to do manual transaction handling thanks to this change. Refs #2277 --- datasette/database.py | 9 +++------ datasette/utils/internal_db.py | 27 +++++++++++++-------------- tests/test_internals_database.py | 10 ++++------ 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d34aac73..4e590d3a 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -123,8 +123,7 @@ class Database: async def execute_write(self, sql, params=None, block=True): def _inner(conn): - with conn: - return conn.execute(sql, params or []) + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block) @@ -132,8 +131,7 @@ class Database: async def execute_write_script(self, sql, block=True): def _inner(conn): - with conn: - return conn.executescript(sql) + return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): results = await self.execute_write_fn(_inner, block=block) @@ -149,8 +147,7 @@ class Database: count += 1 yield param - with conn: - return conn.executemany(sql, count_params(params_seq)), count + return conn.executemany(sql, count_params(params_seq)), count with trace( "sql", database=self.name, sql=sql.strip(), executemany=True diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index dd0d3a9d..dbfcceb4 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,20 +69,19 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - with conn: - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 57e75046..1c155cf3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -501,9 +501,8 @@ async def test_execute_write_has_correctly_prepared_connection(db): @pytest.mark.asyncio async def test_execute_write_fn_block_false(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] task_id = await db.execute_write_fn(write_fn, block=False) @@ -513,9 +512,8 @@ async def test_execute_write_fn_block_false(db): @pytest.mark.asyncio async def test_execute_write_fn_block_true(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] new_count = await db.execute_write_fn(write_fn) From a4fa1ef3bd6a6117118b5cdd64aca2308c21604b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:56:15 -0800 Subject: [PATCH 1241/1866] Release 1.0a10 Refs #2277 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index f5e07ac8..809c434f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a9" +__version__ = "1.0a10" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e567f422..92f198af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a10: + +1.0a10 (2024-02-17) +------------------- + +The only changes in this alpha correspond to the way Datasette handles database transactions. (:issue:`2277`) + +- The :ref:`database.execute_write_fn() ` method has a new ``transaction=True`` parameter. This defaults to ``True`` which means all functions executed using this method are now automatically wrapped in a transaction - previously the functions needed to roll transaction handling on their own, and many did not. +- Pass ``transaction=False`` to ``execute_write_fn()`` if you want to manually handle transactions in your function. +- Several internal Datasette features, including parts of the :ref:`JSON write API `, had been failing to wrap their operations in a transaction. This has been fixed by the new ``transaction=True`` default. + .. _v1_0_a9: 1.0a9 (2024-02-16) From 81629dbeffb5cee9086bc956ce3a9ab7d051f4d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 21:03:41 -0800 Subject: [PATCH 1242/1866] Upgrade GitHub Actions, including PyPI publishing --- .github/workflows/publish.yml | 60 ++++++++++++++--------------------- .github/workflows/test.yml | 13 +++----- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 64a03a77..55fc0eb9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,20 +12,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install -e '.[test]' @@ -36,47 +31,38 @@ jobs: deploy: runs-on: ubuntu-latest needs: [test] + environment: release + permissions: + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + python-version: '3.12' + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | - pip install setuptools wheel twine - - name: Publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + pip install setuptools wheel build + - name: Build run: | - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | python -m pip install -e .[docs] @@ -105,7 +91,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 656b0b1c..3ac8756d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,19 +12,14 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) From 3856a8cb244f1338d2c4bceb76a510022d88ade5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:51:14 -0800 Subject: [PATCH 1243/1866] Consistent Permission denied:, refs #2279 --- datasette/views/database.py | 6 +++--- tests/test_api_write.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 2a8b40cc..56fc6f8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -860,7 +860,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "update-row", resource=database_name ): - return _error(["Permission denied - need update-row"], 403) + return _error(["Permission denied: need update-row"], 403) table_name = data.get("table") if not table_name: @@ -884,7 +884,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "insert-row", resource=database_name ): - return _error(["Permission denied - need insert-row"], 403) + return _error(["Permission denied: need insert-row"], 403) alter = False if rows or row: @@ -897,7 +897,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "alter-table", resource=database_name ): - return _error(["Permission denied - need alter-table"], 403) + return _error(["Permission denied: need alter-table"], 403) alter = True if columns: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 0eb915ba..634f5ee9 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1316,7 +1316,7 @@ async def test_create_table( ["create-table"], {"table": "t", "rows": [{"name": "c"}]}, 403, - ["Permission denied - need insert-row"], + ["Permission denied: need insert-row"], ), # This should work: ( @@ -1330,7 +1330,7 @@ async def test_create_table( ["create-table", "insert-row"], {"table": "t", "rows": [{"id": 1}], "pk": "id", "replace": True}, 403, - ["Permission denied - need update-row"], + ["Permission denied: need update-row"], ), ), ) @@ -1567,7 +1567,7 @@ async def test_create_using_alter_against_existing_table( assert response2.status_code == 403 assert response2.json() == { "ok": False, - "errors": ["Permission denied - need alter-table"], + "errors": ["Permission denied: need alter-table"], } else: assert response2.status_code == 201 From b36a2d8f4b566a3a4902cdaa7a549241e0e8c881 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:55:51 -0800 Subject: [PATCH 1244/1866] Require update-row to use insert replace, closes #2279 --- datasette/views/table.py | 5 +++++ docs/json_api.rst | 4 ++-- tests/test_api_write.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1c187692..6d0d9885 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -485,6 +485,11 @@ class TableInsertView(BaseView): if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + if replace and not await self.ds.permission_allowed( + request.actor, "update-row", resource=(database_name, table_name) + ): + return _error(['Permission denied: need update-row to use "replace"'], 403) + initial_schema = None if alter: # Must have alter-table permission diff --git a/docs/json_api.rst b/docs/json_api.rst index c401d97e..366f74b2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -616,7 +616,7 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: "ignore": true } -Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. This requires the :ref:`permissions_update_row` permission. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. @@ -854,7 +854,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. -* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`permissions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 634f5ee9..6a7ddeb6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -221,6 +221,14 @@ async def test_insert_rows(ds_write, return_rows): 400, ['Cannot use "ignore" and "replace" at the same time'], ), + ( + # Replace is not allowed if you don't have update-row + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "replace": True}, + "insert-but-not-update", + 403, + ['Permission denied: need update-row to use "replace"'], + ), ( "/data/docs/-/insert", {"rows": [{"title": "Test"}], "invalid_param": True}, From 392ca2e24cc93a3918d07718f40524857d626d14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 13:40:48 -0800 Subject: [PATCH 1245/1866] Improvements to table column cog menu display, closes #2263 - Repositions if menu would cause a horizontal scrollbar - Arrow tip on menu now attempts to align with cog icon on column --- datasette/static/table.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/datasette/static/table.js b/datasette/static/table.js index 778457c5..0c54a472 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -217,6 +217,17 @@ const initDatasetteTable = function (manager) { menuList.appendChild(menuItem); }); + // Measure width of menu and adjust position if too far right + const menuWidth = menu.offsetWidth; + const windowWidth = window.innerWidth; + if (menuLeft + menuWidth > windowWidth) { + menu.style.left = windowWidth - menuWidth - 20 + "px"; + } + // Align menu .hook arrow with the column cog icon + const hook = menu.querySelector('.hook'); + const icon = th.querySelector('.dropdown-menu-icon'); + const iconRect = icon.getBoundingClientRect(); + hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; } var svg = document.createElement("div"); From 27409a78929b4baa017cce2cc0ca636603ed6d37 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:01:55 -0800 Subject: [PATCH 1246/1866] Fix for hook position in wide column names, refs #2263 --- datasette/static/table.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index 0c54a472..4f81b2e5 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -227,7 +227,15 @@ const initDatasetteTable = function (manager) { const hook = menu.querySelector('.hook'); const icon = th.querySelector('.dropdown-menu-icon'); const iconRect = icon.getBoundingClientRect(); - hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; + const hookLeft = (iconRect.left - menuLeft + 1) + 'px'; + hook.style.left = hookLeft; + // Move the whole menu right if the hook is too far right + const menuRect = menu.getBoundingClientRect(); + if (iconRect.right > menuRect.right) { + menu.style.left = (iconRect.right - menuWidth) + 'px'; + // And move hook tip as well + hook.style.left = (menuWidth - 13) + 'px'; + } } var svg = document.createElement("div"); From 26300738e3c6e7ad515bd513063f57249a05000a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:17:37 -0800 Subject: [PATCH 1247/1866] Fixes for permissions debug page, closes #2278 --- datasette/templates/permissions_debug.html | 10 +++++----- datasette/views/special.py | 17 +++++++++-------- tests/test_permissions.py | 8 ++++++++ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 36a12acc..5a5c1aa6 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -57,7 +57,7 @@ textarea {

    @@ -71,19 +71,19 @@ textarea { +{% endif %} + {% endblock %} diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 9c6bbde0..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -31,6 +31,12 @@ class Urls: db = self.ds.get_database(database) return self.path(tilde_encode(db.route), format=format) + def database_query(self, database, sql, format=None): + path = f"{self.database(database)}/-/query?" + urllib.parse.urlencode( + {"sql": sql} + ) + return self.path(path, format=format) + def table(self, database, table, format=None): path = f"{self.database(database)}/{tilde_encode(table)}" if format is not None: diff --git a/datasette/views/table.py b/datasette/views/table.py index d71efeb0..ea044b36 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -929,6 +929,7 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), + count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1280,6 +1281,9 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) + async def extra_count_sql(): + return count_sql + async def extra_count(): "Total count of rows matching these filters" # Calculate the total count for this query @@ -1299,8 +1303,11 @@ async def table_view_data( # Otherwise run a select count(*) ... if count_sql and count is None and not nocount: + count_sql_limited = ( + f"select count(*) from (select * {from_sql} limit 10001)" + ) try: - count_rows = list(await db.execute(count_sql, from_sql_params)) + count_rows = list(await db.execute(count_sql_limited, from_sql_params)) count = count_rows[0][0] except QueryInterrupted: pass @@ -1615,6 +1622,7 @@ async def table_view_data( "facet_results", "facets_timed_out", "count", + "count_sql", "human_description_en", "next_url", "metadata", @@ -1647,6 +1655,7 @@ async def table_view_data( registry = Registry( extra_count, + extra_count_sql, extra_facet_results, extra_facets_timed_out, extra_suggested_facets, From dc288056b81a3635bdb02a6d0121887db2720e5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 21 Aug 2024 19:56:02 -0700 Subject: [PATCH 1346/1866] Better handling of errors for count all button, refs #2408 --- datasette/templates/table.html | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 187f0143..7246ff5d 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -42,7 +42,7 @@ {% if count or human_description_en %}

    {% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows - {% if allow_execute_sql and query.sql %} count all rows{% endif %} + {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %}

    @@ -180,7 +180,7 @@ document.addEventListener('DOMContentLoaded', function() { const countLink = document.querySelector('a.count-sql'); if (countLink) { - countLink.addEventListener('click', function(ev) { + countLink.addEventListener('click', async function(ev) { ev.preventDefault(); // Replace countLink with span with same style attribute const span = document.createElement('span'); @@ -189,14 +189,23 @@ document.addEventListener('DOMContentLoaded', function() { countLink.replaceWith(span); countLink.setAttribute('disabled', 'disabled'); let url = countLink.href.replace(/(\?|$)/, '.json$1'); - fetch(url) - .then(response => response.json()) - .then(data => { - const count = data['rows'][0]['count(*)']; - const formattedCount = count.toLocaleString(); - span.closest('h3').textContent = formattedCount + ' rows'; - }) - .catch(error => countLink.textContent = 'error'); + try { + const response = await fetch(url); + console.log({response}); + const data = await response.json(); + console.log({data}); + if (!response.ok) { + console.log('throw error'); + throw new Error(data.title || data.error); + } + const count = data['rows'][0]['count(*)']; + const formattedCount = count.toLocaleString(); + span.closest('h3').textContent = formattedCount + ' rows'; + } catch (error) { + console.log('Update', span, 'with error message', error); + span.textContent = error.message; + span.style.color = 'red'; + } }); } }); From 92c4d41ca605e0837a2711ee52fde9cf1eea74d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Sep 2024 17:20:41 -0700 Subject: [PATCH 1347/1866] results.dicts() method, closes #2414 --- datasette/database.py | 3 +++ datasette/views/row.py | 3 +-- datasette/views/table.py | 2 +- docs/internals.rst | 3 +++ tests/test_api_write.py | 23 +++++++++-------------- tests/test_internals_database.py | 11 +++++++++++ 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index da0ab1de..a2e899bc 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -677,6 +677,9 @@ class Results: else: raise MultipleValues + def dicts(self): + return [dict(row) for row in self.rows] + def __iter__(self): return iter(self.rows) diff --git a/datasette/views/row.py b/datasette/views/row.py index d802994e..f374fd94 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -277,8 +277,7 @@ class RowUpdateView(BaseView): results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) - rows = list(results.rows) - result["row"] = dict(rows[0]) + result["row"] = results.dicts()[0] await self.ds.track_event( UpdateRowEvent( diff --git a/datasette/views/table.py b/datasette/views/table.py index ea044b36..82dab613 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -558,7 +558,7 @@ class TableInsertView(BaseView): ), args, ) - result["rows"] = [dict(r) for r in fetched_rows.rows] + result["rows"] = fetched_rows.dicts() else: result["rows"] = rows # We track the number of rows requested, but do not attempt to show which were actually diff --git a/docs/internals.rst b/docs/internals.rst index 4289c815..facbc224 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1093,6 +1093,9 @@ The ``Results`` object also has the following properties and methods: ``.rows`` - list of ``sqlite3.Row`` This property provides direct access to the list of rows returned by the database. You can access specific rows by index using ``results.rows[0]``. +``.dicts()`` - list of ``dict`` + This method returns a list of Python dictionaries, one for each row. + ``.first()`` - row or None Returns the first row in the results, or ``None`` if no rows were returned. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9c2b9b45..04e61261 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -58,8 +58,8 @@ async def test_insert_row(ds_write, content_type): assert response.status_code == 201 assert response.json()["ok"] is True assert response.json()["rows"] == [expected_row] - rows = (await ds_write.get_database("data").execute("select * from docs")).rows - assert dict(rows[0]) == expected_row + rows = (await ds_write.get_database("data").execute("select * from docs")).dicts() + assert rows[0] == expected_row # Analytics event event = last_event(ds_write) assert event.name == "insert-rows" @@ -118,12 +118,9 @@ async def test_insert_rows(ds_write, return_rows): assert not event.ignore assert not event.replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} @@ -469,12 +466,10 @@ async def test_insert_ignore_replace( assert event.ignore == ignore assert event.replace == replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() + assert actual_rows == expected_rows assert response.json()["ok"] is True if should_return: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0020668a..edfc6bc7 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -40,6 +40,17 @@ async def test_results_bool(db, expected): assert bool(results) is expected +@pytest.mark.asyncio +async def test_results_dicts(db): + results = await db.execute("select pk, name from roadside_attractions") + assert results.dicts() == [ + {"pk": 1, "name": "The Mystery Spot"}, + {"pk": 2, "name": "Winchester Mystery House"}, + {"pk": 3, "name": "Burlingame Museum of PEZ Memorabilia"}, + {"pk": 4, "name": "Bigfoot Discovery Museum"}, + ] + + @pytest.mark.parametrize( "query,expected", [ From 2170269258d1de38f4e518aa3e55e6b3ed202841 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Sep 2024 08:37:26 -0700 Subject: [PATCH 1348/1866] New .core CSS class for inputs and buttons * Initial .core input/button classes, refs #2415 * Docs for the new .core CSS class, refs #2415 * Applied .core class everywhere that needs it, closes #2415 --- datasette/static/app.css | 33 +++++++++++++++------- datasette/templates/allow_debug.html | 2 +- datasette/templates/api_explorer.html | 4 +-- datasette/templates/create_token.html | 2 +- datasette/templates/database.html | 2 +- datasette/templates/logout.html | 2 +- datasette/templates/messages_debug.html | 2 +- datasette/templates/permissions_debug.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/table.html | 4 +-- docs/custom_templates.rst | 9 ++++++ docs/writing_plugins.rst | 3 +- tests/test_permissions.py | 2 +- 13 files changed, 46 insertions(+), 23 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 562d6adb..f975f0ad 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -528,8 +528,11 @@ label.sort_by_desc { pre#sql-query { margin-bottom: 1em; } -form input[type=text], -form input[type=search] { + +.core input[type=text], +input.core[type=text], +.core input[type=search], +input.core[type=search] { border: 1px solid #ccc; border-radius: 3px; width: 60%; @@ -540,17 +543,25 @@ form input[type=search] { } /* Stop Webkit from styling search boxes in an inconsistent way */ /* https://css-tricks.com/webkit-html5-search-inputs/ comments */ -input[type=search] { +.core input[type=search], +input.core[type=search] { -webkit-appearance: textfield; } -input[type="search"]::-webkit-search-decoration, -input[type="search"]::-webkit-search-cancel-button, -input[type="search"]::-webkit-search-results-button, -input[type="search"]::-webkit-search-results-decoration { +.core input[type="search"]::-webkit-search-decoration, +input.core[type="search"]::-webkit-search-decoration, +.core input[type="search"]::-webkit-search-cancel-button, +input.core[type="search"]::-webkit-search-cancel-button, +.core input[type="search"]::-webkit-search-results-button, +input.core[type="search"]::-webkit-search-results-button, +.core input[type="search"]::-webkit-search-results-decoration, +input.core[type="search"]::-webkit-search-results-decoration { display: none; } -form input[type=submit], form button[type=button] { +.core input[type=submit], +.core button[type=button], +input.core[type=submit], +button.core[type=button] { font-weight: 400; cursor: pointer; text-align: center; @@ -563,14 +574,16 @@ form input[type=submit], form button[type=button] { border-radius: .25rem; } -form input[type=submit] { +.core input[type=submit], +input.core[type=submit] { color: #fff; background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); border-color: #007bff; -webkit-appearance: button; } -form button[type=button] { +.core button[type=button], +button.core[type=button] { color: #007bff; background-color: #fff; border-color: #007bff; diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 04181531..610417d2 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -35,7 +35,7 @@ p.message-warning {

    Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

    - +

    diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 109fb1e9..dc393c20 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -19,7 +19,7 @@

    GET - +
    @@ -29,7 +29,7 @@
    POST - +
    diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index 2be98d38..409fb8a9 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -39,7 +39,7 @@ {% endfor %} {% endif %} - +

    diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html index 4c4a7d11..c8fc642a 100644 --- a/datasette/templates/logout.html +++ b/datasette/templates/logout.html @@ -8,7 +8,7 @@

    You are logged in as {{ display_actor(actor) }}

    - +
    diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index e0ab9a40..2940cd69 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -8,7 +8,7 @@

    Set a message:

    - +
    diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 5a5c1aa6..83891181 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -47,7 +47,7 @@ textarea {

    This tool lets you simulate an actor and a permission check for that actor.

    - +

    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f7c8d0a3..a6e9a3aa 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -36,7 +36,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 7246ff5d..c9e0e87b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -48,7 +48,7 @@

    {% endif %} - + {% if supports_search %}
    {% endif %} @@ -152,7 +152,7 @@ object {% endif %}

    - +

    CSV options: diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 534d8b33..8cc40f0f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -83,6 +83,15 @@ database column they are representing, for example: +.. _customization_css: + +Writing custom CSS +~~~~~~~~~~~~~~~~~~ + +Custom templates need to take Datasette's default CSS into account. The pattern portfolio at ``/-/patterns`` (`example here `__) is a useful reference for understanding the available CSS classes. + +The ``core`` class is particularly useful - you can apply this directly to a ```` or `` +

    +

    + + + + +

    + + +{% if queries %} +
      + {% for query in queries %} +
    • + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.is_write %}Writable{% endif %} + {% if query.is_published %}Published{% endif %} +
      • + {% endfor %} +
    +{% else %} +

    No queries found.

    +{% endif %} + +{% if next_url %} +

    Next page

    +{% endif %} + +{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index ed38189b..edbc315e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -92,24 +92,14 @@ class DatabaseView(View): tables = await get_tables(datasette, request, db, allowed_dict) - # Get allowed queries using the new permission system - allowed_query_page = await datasette.allowed_resources( - "view-query", - request.actor, - parent=database, - include_is_private=True, - limit=1000, + queries_page = await datasette.list_queries( + database, + actor=request.actor, + limit=20, + include_private=True, ) - - # Build canned_queries list by looking up each allowed query - all_queries = await datasette.get_canned_queries(database, request.actor) - canned_queries = [] - for query_resource in allowed_query_page.resources: - query_name = query_resource.child - if query_name in all_queries: - canned_queries.append( - dict(all_queries[query_name], private=query_resource.private) - ) + canned_queries = queries_page["queries"] + queries_more = queries_page["has_more"] async def database_actions(): links = [] @@ -141,6 +131,7 @@ class DatabaseView(View): "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, "queries": canned_queries, + "queries_more": queries_more, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -174,6 +165,7 @@ class DatabaseView(View): hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, queries=canned_queries, + queries_more=queries_more, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -222,6 +214,9 @@ class DatabaseContext(Context): hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of canned query objects"}) + queries_more: bool = field( + metadata={"help": "Boolean indicating if more saved queries are available"} + ) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -474,6 +469,31 @@ def _as_bool(value): return bool(value) +def _as_optional_bool(value, name): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "t", "yes", "on"}: + return True + if lowered in {"0", "false", "f", "no", "off"}: + return False + raise QueryValidationError("{} must be 0 or 1".format(name)) + + +def _query_list_limit(value): + if value in (None, ""): + return 50 + try: + return min(max(1, int(value)), 1000) + except ValueError as ex: + raise QueryValidationError("_size must be an integer") from ex + + def _derived_query_parameters(sql): parameters = [] seen = set() @@ -949,19 +969,66 @@ class QueryListView(BaseView): async def get(self, request): db = await self.ds.resolve_database(request) - page = await self.ds.allowed_resources( - "view-query", - request.actor, - parent=db.name, - limit=1000, + format_ = request.url_vars.get("format") or "html" + try: + limit = _query_list_limit(request.args.get("_size")) + is_write = _as_optional_bool(request.args.get("is_write"), "is_write") + is_published = _as_optional_bool( + request.args.get("is_published"), "is_published" + ) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + page = await self.ds.list_queries( + db.name, + actor=request.actor, + limit=limit, + cursor=request.args.get("_next"), + q=request.args.get("q") or None, + is_write=is_write, + is_published=is_published, + source=request.args.get("source") or None, + owner_id=request.args.get("owner_id") or None, + include_private=True, + ) + next_url = None + if page["next"]: + pairs = [ + (key, value) + for key, value in parse_qsl( + request.query_string, keep_blank_values=True + ) + if key != "_next" + ] + pairs.append(("_next", page["next"])) + next_url = "{}?{}".format( + self.ds.urls.database(db.name) + "/-/queries", + urlencode(pairs), + ) + + data = { + "ok": True, + "database": db.name, + "queries": page["queries"], + "next": page["next"], + "next_url": next_url, + "has_more": page["has_more"], + "limit": page["limit"], + "filters": { + "q": request.args.get("q") or "", + "is_write": request.args.get("is_write") or "", + "is_published": request.args.get("is_published") or "", + "source": request.args.get("source") or "", + "owner_id": request.args.get("owner_id") or "", + }, + } + if format_ == "json": + return Response.json(data) + return await self.render( + ["query_list.html"], + request, + data, ) - all_queries = await self.ds.get_queries(db.name) - queries = [ - all_queries[resource.child] - for resource in page.resources - if resource.child in all_queries - ] - return Response.json({"ok": True, "database": db.name, "queries": queries}) class QueryCreateView(BaseView): diff --git a/docs/json_api.rst b/docs/json_api.rst index e4c9e86e..ece430c2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -510,7 +510,7 @@ Datasette provides a write API for JSON data. This is a POST-only API that requi Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries`` returns saved query definitions the actor can view. +``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index a58ace70..671fc29c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` lists query definitions the actor can view or manage, probably paginated. +- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -353,9 +353,21 @@ await datasette.update_query( await datasette.remove_query(database, name, source=None) await datasette.get_query(database, name) -await datasette.get_queries(database) +await datasette.list_queries( + database, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_published=None, + source=None, + owner_id=None, +) ``` +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. + `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": ```python @@ -380,6 +392,8 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. + ## Dedicated create query UI Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. diff --git a/tests/test_queries.py b/tests/test_queries.py index df4131b9..dd906faf 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -7,6 +7,20 @@ from datasette.resources import DatabaseResource, QueryResource from datasette.utils.asgi import Forbidden +async def add_numbered_queries(ds, database, count): + for i in range(1, count + 1): + await ds.add_query( + database, + "demo_query_{:02d}".format(i), + "select {} as query_number".format(i), + title="Demo query {:02d}".format(i), + description="Seeded demo query number {:02d}".format(i), + is_published=True, + source="user", + owner_id="root", + ) + + @pytest.mark.asyncio async def test_queries_internal_table_schema(): ds = Datasette(memory=True) @@ -96,11 +110,15 @@ async def test_add_get_and_remove_query(): "on_error_redirect": None, } - assert await ds.get_queries("data") == {"top_customers": query} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [query] + assert queries_page["next"] is None await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None - assert await ds.get_queries("data") == {} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [] + assert queries_page["next"] is None @pytest.mark.asyncio @@ -238,6 +256,24 @@ async def test_unpublished_query_requires_execute_sql_but_published_does_not(): ) +@pytest.mark.asyncio +async def test_database_page_query_preview_is_limited(): + ds = Datasette(memory=True) + ds.add_memory_database("query_preview", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + html_response = await ds.client.get("/data") + json_response = await ds.client.get("/data.json") + + assert html_response.status_code == 200 + assert "Demo query 20" in html_response.text + assert "Demo query 21" not in html_response.text + assert 'href="/data/-/queries"' in html_response.text + assert len(json_response.json()["queries"]) == 20 + assert json_response.json()["queries_more"] is True + + @pytest.mark.asyncio async def test_query_actions_are_registered(): ds = Datasette() @@ -347,21 +383,78 @@ async def test_query_list_and_definition_api(): ds.root_enabled = True ds.add_memory_database("query_list_api", name="data") await ds.invoke_startup() - await ds.add_query("data", "listed", "select 1", title="Listed", is_published=True) + await add_numbered_queries(ds, "data", 12) list_response = await ds.client.get( - "/data/-/queries", + "/data/-/queries.json?_size=5", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/data/-/queries.json?_size=5&_next={}".format(list_response.json()["next"]), actor={"id": "root"}, ) definition_response = await ds.client.get( - "/data/listed/-/definition", + "/data/demo_query_01/-/definition", actor={"id": "root"}, ) assert list_response.status_code == 200 - assert list_response.json()["queries"][0]["name"] == "listed" + assert [query["name"] for query in list_response.json()["queries"]] == [ + "demo_query_01", + "demo_query_02", + "demo_query_03", + "demo_query_04", + "demo_query_05", + ] + assert list_response.json()["next"] + assert [query["name"] for query in next_response.json()["queries"]] == [ + "demo_query_06", + "demo_query_07", + "demo_query_08", + "demo_query_09", + "demo_query_10", + ] assert definition_response.status_code == 200 - assert definition_response.json()["query"]["title"] == "Listed" + assert definition_response.json()["query"]["title"] == "Demo query 01" + + +@pytest.mark.asyncio +async def test_query_list_search_filter_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 3) + await ds.add_query( + "data", + "private_query", + "select 'private'", + title="Private query", + is_published=False, + source="user", + owner_id="root", + ) + + html_response = await ds.client.get( + "/data/-/queries?q=02", + actor={"id": "root"}, + ) + json_response = await ds.client.get( + "/data/-/queries.json?q=02", + actor={"id": "root"}, + ) + filtered_response = await ds.client.get( + "/data/-/queries.json?is_published=0", + actor={"id": "root"}, + ) + + assert html_response.status_code == 200 + assert "Demo query 02" in html_response.text + assert "Demo query 01" not in html_response.text + assert json_response.json()["queries"][0]["name"] == "demo_query_02" + assert [query["name"] for query in filtered_response.json()["queries"]] == [ + "private_query" + ] @pytest.mark.asyncio From 310c36ae94c54d4b859925d4977554c2a2618534 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:18:36 -0700 Subject: [PATCH 1746/1866] Limit database query preview to five Refs #2735 --- datasette/views/database.py | 2 +- queries-plan.md | 2 +- tests/test_canned_queries.py | 35 ++++++++++++++++++++++++++++++----- tests/test_queries.py | 6 +++--- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index edbc315e..353cfcf2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -95,7 +95,7 @@ class DatabaseView(View): queries_page = await datasette.list_queries( database, actor=request.actor, - limit=20, + limit=5, include_private=True, ) canned_queries = queries_page["queries"] diff --git a/queries-plan.md b/queries-plan.md index 671fc29c..82ef3260 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. ## Dedicated create query UI diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index c46fd86f..a9d22036 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -248,10 +248,9 @@ def test_json_response(canned_write_client, headers, body, querystring): def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth only shows three queries - query_names = { - q["name"] for q in canned_write_client.get("/data.json").json["queries"] - } + # Without auth shows the five public queries + anon_response = canned_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} assert query_names == { "add_name_specify_id_with_error_in_on_success_message_sql", "update_name", @@ -259,8 +258,9 @@ def test_canned_query_permissions_on_database_page(canned_write_client): "canned_read", "add_name", } + assert anon_response.json["queries_more"] is False - # With auth shows four + # With auth the database page preview shows the first five queries response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, @@ -273,6 +273,31 @@ def test_canned_query_permissions_on_database_page(canned_write_client): ], key=lambda q: q["name"], ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "canned_read", "private": False}, + {"name": "delete_name", "private": True}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = canned_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, diff --git a/tests/test_queries.py b/tests/test_queries.py index dd906faf..2b46e00f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -267,10 +267,10 @@ async def test_database_page_query_preview_is_limited(): json_response = await ds.client.get("/data.json") assert html_response.status_code == 200 - assert "Demo query 20" in html_response.text - assert "Demo query 21" not in html_response.text + assert "Demo query 05" in html_response.text + assert "Demo query 06" not in html_response.text assert 'href="/data/-/queries"' in html_response.text - assert len(json_response.json()["queries"]) == 20 + assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True From 6eee6c81e8c21737e2391af55baf24866429038d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:24:42 -0700 Subject: [PATCH 1747/1866] Add global query browser Refs #2735 --- datasette/app.py | 57 +++++++++++++++++++----- datasette/templates/query_list.html | 11 +++-- datasette/views/database.py | 27 ++++++++++-- docs/json_api.rst | 3 +- queries-plan.md | 6 +-- tests/test_queries.py | 67 +++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 22 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bdbf9389..c047fde9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -52,6 +52,7 @@ from .views.database import ( QueryCreateView, QueryDeleteView, QueryDefinitionView, + GlobalQueryListView, QueryInsertView, QueryListView, QueryUpdateView, @@ -1290,7 +1291,7 @@ class Datasette: async def list_queries( self, - database, + database=None, *, actor=None, limit=50, @@ -1310,16 +1311,40 @@ class Datasette: include_is_private=include_private, ) params = dict(allowed_params) - params.update({"query_database": database, "limit": limit + 1}) + params.update({"limit": limit + 1}) sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = ["q.database_name = :query_database"] + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" if cursor: try: components = urlsafe_components(cursor) except ValueError: components = [] - if len(components) == 2: + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key @@ -1368,13 +1393,14 @@ class Datasette: ON allowed.parent = q.database_name AND allowed.child = q.name WHERE {where} - ORDER BY sort_key, q.name + ORDER BY {order_by} LIMIT :limit """.format( allowed_sql=allowed_sql, private_select=private_select, sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses), + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, ), params, ) @@ -1394,10 +1420,17 @@ class Datasette: next_token = None if has_more and rows: last_row = rows[-1] - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) return { "queries": queries, "next": next_token, @@ -2651,6 +2684,10 @@ class Datasette: JumpView.as_view(self), r"/-/jump(\.(?Pjson))?$", ) + add_route( + GlobalQueryListView.as_view(self), + r"/-/queries(\.(?Pjson))?$", + ) add_route( InstanceSchemaView.as_view(self), r"/-/schema(\.(?Pjson|md))?$", diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index ef5da0d5..af974550 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -1,8 +1,8 @@ {% extends "base.html" %} -{% block title %}{{ database }}: queries{% endblock %} +{% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} -{% block body_class %}query-list db-{{ database|to_css_class }}{% endblock %} +{% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} {{ crumbs.nav(request=request, database=database) }} @@ -12,7 +12,7 @@

    Queries

    -
    +

    @@ -38,7 +38,10 @@

      {% for query in queries %}
    • - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if show_database %} + {{ query.database }}: + {% endif %} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.is_write %}Writable{% endif %} {% if query.is_published %}Published{% endif %}
      • diff --git a/datasette/views/database.py b/datasette/views/database.py index 353cfcf2..1576b6a9 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -967,8 +967,14 @@ class ExecuteWriteView(BaseView): class QueryListView(BaseView): name = "query-list" + async def database_name(self, request): + return (await self.ds.resolve_database(request)).name + + def query_list_path(self, database): + return self.ds.urls.database(database) + "/-/queries" + async def get(self, request): - db = await self.ds.resolve_database(request) + database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: limit = _query_list_limit(request.args.get("_size")) @@ -980,7 +986,7 @@ class QueryListView(BaseView): return _error([ex.message], ex.status) page = await self.ds.list_queries( - db.name, + database, actor=request.actor, limit=limit, cursor=request.args.get("_next"), @@ -991,6 +997,7 @@ class QueryListView(BaseView): owner_id=request.args.get("owner_id") or None, include_private=True, ) + query_list_path = self.query_list_path(database) next_url = None if page["next"]: pairs = [ @@ -1002,18 +1009,20 @@ class QueryListView(BaseView): ] pairs.append(("_next", page["next"])) next_url = "{}?{}".format( - self.ds.urls.database(db.name) + "/-/queries", + query_list_path, urlencode(pairs), ) data = { "ok": True, - "database": db.name, + "database": database, "queries": page["queries"], "next": page["next"], "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "query_list_path": query_list_path, + "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1031,6 +1040,16 @@ class QueryListView(BaseView): ) +class GlobalQueryListView(QueryListView): + name = "global-query-list" + + async def database_name(self, request): + return None + + def query_list_path(self, database): + return self.ds.urls.path("/-/queries") + + class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False diff --git a/docs/json_api.rst b/docs/json_api.rst index ece430c2..f44a39fe 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,12 +505,13 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _GlobalQueryListView: .. _QueryListView: Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns saved query definitions across every database that the actor can view. ``GET //-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index 82ef3260..a708e887 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. +- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -366,7 +366,7 @@ await datasette.list_queries( ) ``` -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. ## Dedicated create query UI diff --git a/tests/test_queries.py b/tests/test_queries.py index 2b46e00f..bc04bb51 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -457,6 +457,73 @@ async def test_query_list_search_filter_and_html(): ] +@pytest.mark.asyncio +async def test_global_query_list_api_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_global_alpha", name="alpha") + ds.add_memory_database("query_list_global_beta", name="beta") + await ds.invoke_startup() + await ds.add_query( + "alpha", + "alpha_first", + "select 1", + title="Alpha first", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "alpha", + "alpha_second", + "select 2", + title="Alpha second", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "beta", + "beta_first", + "select 3", + title="Beta first", + is_published=True, + source="user", + owner_id="root", + ) + + list_response = await ds.client.get( + "/-/queries.json?_size=2", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/-/queries.json?_size=2&_next={}".format(list_response.json()["next"]), + actor={"id": "root"}, + ) + html_response = await ds.client.get( + "/-/queries?q=Beta", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert [ + (query["database"], query["name"]) for query in list_response.json()["queries"] + ] == [ + ("alpha", "alpha_first"), + ("alpha", "alpha_second"), + ] + assert list_response.json()["next"] + assert [ + (query["database"], query["name"]) for query in next_response.json()["queries"] + ] == [ + ("beta", "beta_first"), + ] + assert html_response.status_code == 200 + assert 'href="/beta">beta:' in html_response.text + assert "Beta first" in html_response.text + assert "Alpha first" not in html_response.text + + @pytest.mark.asyncio async def test_query_insert_api_publish_requires_publish_query(): ds = Datasette( From f0b59971f7c8c0f4435a18b4f4e9c8053c2683fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:39:56 -0700 Subject: [PATCH 1748/1866] Delete unnecessary test --- tests/test_utils_sql_analysis.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index c82fb04f..5730cd0d 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -169,13 +169,6 @@ def test_analyze_attached_database_tables(conn): } -def test_analyze_invalid_sql_cleans_up_authorizer(conn): - with pytest.raises(sqlite3.OperationalError): - analyze_sql_tables(conn, "insert into missing_table values (1)") - - conn.execute("select name from dogs").fetchall() - - def test_analyze_clears_authorizer_on_error(): class FakeConnection: def __init__(self): From 2b5b4ed66b86bae0080e9d8f4881cad8e57bbdb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:11:08 -0700 Subject: [PATCH 1749/1866] Much improved "Write to this database" UI - Start with a template option, letting you pick table and operation - SQL textarea defaults to 4 empty lines at start - Query operations table is simpler and looks nicer Refs #2742 --- datasette/templates/execute_write.html | 240 +++++++++++++++++++++++-- datasette/views/database.py | 13 +- tests/test_queries.py | 32 +++- 3 files changed, 271 insertions(+), 14 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5b4f30d9..90845910 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -1,10 +1,80 @@ {% extends "base.html" %} -{% block title %}Execute write SQL{% endblock %} +{% block title %}Write to this database{% endblock %} {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} + {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -15,13 +85,34 @@ {% block content %} -

      Execute write SQL

      +

      Write to this database

      + +

      Execute SQL to insert, update or delete rows in this database.

      {% if execution_message %}

      {{ execution_message }}

      {% endif %} + {% if write_template_tables %} +
      +
      + Start with a template +

      + + + + + +

      +
      +
      + {% endif %} +

      {% if parameter_names %} @@ -31,30 +122,28 @@ {% endfor %} {% endif %} -

      Analysis

      +

      Query operations

      {% if analysis_error %}

      {{ analysis_error }}

      {% elif analysis_rows %} -
      +
      - + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} @@ -66,6 +155,133 @@

      + + {% include "_codemirror_foot.html" %} +{% if write_template_tables %} + +{% endif %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 1576b6a9..fb3bdfdb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -830,6 +830,13 @@ class ExecuteWriteView(BaseView): parameter_values = parameter_values or {} parameter_names = [] analysis_rows = [] + table_columns = await _table_columns(self.ds, db.name) + hidden_table_names = set(await db.hidden_table_names()) + write_template_tables = { + table: columns + for table, columns in table_columns.items() + if columns and table not in hidden_table_names + } if sql and analysis_error is None: try: parameter_names = _derived_query_parameters(sql) @@ -858,7 +865,9 @@ class ExecuteWriteView(BaseView): "parameter_names": parameter_names, "parameter_values": parameter_values, "analysis_error": analysis_error, - "analysis_rows": analysis_rows, + "analysis_rows": [ + row for row in analysis_rows if row["operation"] != "read" + ], "execution_message": execution_message, "execution_ok": execution_ok, "execute_disabled": bool( @@ -866,6 +875,8 @@ class ExecuteWriteView(BaseView): or analysis_error or any(row["allowed"] is False for row in analysis_rows) ), + "table_columns": table_columns, + "write_template_tables": write_template_tables, }, ) response.status = status diff --git a/tests/test_queries.py b/tests/test_queries.py index bc04bb51..684454fc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -690,6 +690,14 @@ async def test_execute_write_get_prepopulates_without_executing(): ds.root_enabled = True db = ds.add_memory_database("execute_write_get", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + await db.execute_write("create table log (message text)") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + update cats set name = new.name where id = new.id; + insert into log (message) values (new.name); + end + """) await ds.invoke_startup() response = await ds.client.get( @@ -700,11 +708,33 @@ async def test_execute_write_get_prepopulates_without_executing(): assert response.status_code == 200 assert response.headers["content-security-policy"] == "frame-ancestors 'none'" assert response.headers["x-frame-options"] == "DENY" - assert "Execute write SQL" in response.text + assert "Write to this database" in response.text + assert ( + "Execute SQL to insert, update or delete rows in this database." + in response.text + ) + assert "

      Query operations

      " in response.text + assert "Start with a template" in response.text + assert '' in response.text + assert 'data-sql-template="insert"' in response.text + assert 'data-sql-template="update"' in response.text + assert 'data-sql-template="delete"' in response.text + assert '
      Operation Database Tablerequired permissionRequired permission AllowedSource
      {{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
      ' in response.text + assert '' in response.text + assert "" in response.text + assert "" in response.text + assert "" not in response.text assert 'action="/data/-/execute-write"' in response.text assert "insert into dogs (name) values ('Cleo')" in response.text assert (await db.execute("select count(*) from dogs")).first()[0] == 0 + empty_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + ) + assert '' in empty_response.text + assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): From 1bce34a33869709e1dea21b6182327a105895285 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:22:24 -0700 Subject: [PATCH 1750/1866] If just a single insert, link to row page Refs #2742 --- datasette/templates/execute_write.html | 2 +- datasette/views/database.py | 49 ++++++++++++++++++++++++++ tests/test_queries.py | 42 ++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 90845910..705181d8 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -90,7 +90,7 @@

      Execute SQL to insert, update or delete rows in this database.

      {% if execution_message %} -

      {{ execution_message }}

      +

      {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

      {% endif %}
      diff --git a/datasette/views/database.py b/datasette/views/database.py index fb3bdfdb..2b3920f7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -18,8 +18,10 @@ from datasette.utils import ( await_me_maybe, call_with_supported_arguments, named_parameters as derive_named_parameters, + escape_sqlite, format_bytes, make_slot_function, + path_from_row_pks, tilde_decode, to_css_class, validate_sql_select, @@ -678,6 +680,43 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _inserted_row_url(datasette, db, analysis, cursor): + if cursor.rowcount != 1: + return None + lastrowid = getattr(cursor, "lastrowid", None) + if lastrowid is None: + return None + direct_inserts = [ + access + for access in analysis.table_accesses + if access.operation == "insert" + and access.source is None + and access.database == db.name + ] + if len(direct_inserts) != 1: + return None + table = direct_inserts[0].table + pks = await db.primary_keys(table) + use_rowid = not pks + select = ( + "rowid" + if use_rowid + else ", ".join(escape_sqlite(primary_key) for primary_key in pks) + ) + try: + result = await db.execute( + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + [lastrowid], + ) + except sqlite3.DatabaseError: + return None + row = result.first() + if row is None: + return None + row_path = path_from_row_pks(row, pks, use_rowid) + return datasette.urls.row(db.name, table, row_path) + + def _apply_query_data_types(data): typed = dict(data) for key in ("hide_sql", "is_published"): @@ -824,10 +863,12 @@ class ExecuteWriteView(BaseView): analysis=None, analysis_error=None, execution_message=None, + execution_links=None, execution_ok=None, status=200, ): parameter_values = parameter_values or {} + execution_links = execution_links or [] parameter_names = [] analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) @@ -869,6 +910,7 @@ class ExecuteWriteView(BaseView): row for row in analysis_rows if row["operation"] != "read" ], "execution_message": execution_message, + "execution_links": execution_links, "execution_ok": execution_ok, "execute_disabled": bool( (not sql) @@ -964,6 +1006,12 @@ class ExecuteWriteView(BaseView): ) ) + inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor) + execution_links = ( + [{"href": inserted_row_url, "label": "View row"}] + if inserted_row_url + else [] + ) return await self._render_form( request, db, @@ -971,6 +1019,7 @@ class ExecuteWriteView(BaseView): parameter_values={name: params.get(name, "") for name in parameter_names}, analysis=analysis, execution_message=message, + execution_links=execution_links, execution_ok=True, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index 684454fc..ed981ee7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -849,6 +849,48 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_links_to_inserted_row(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_insert_link", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table log (id integer primary key, message text)") + await db.execute_write("insert into log (message) values ('existing')") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + insert into log (message) values (new.name); + end + """) + await ds.invoke_startup() + + insert_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "insert into dogs (name) values (:name)", + "name": "Cleo", + }, + ) + update_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update dogs set name = :name where id = :id", + "name": "Cleo 2", + "id": "1", + }, + ) + + assert insert_response.status_code == 200 + assert "Query executed, 1 row affected" in insert_response.text + assert 'View row' in insert_response.text + assert "/data/log/2" not in insert_response.text + assert update_response.status_code == 200 + assert "Query executed, 1 row affected" in update_response.text + assert "View row" not in update_response.text + + @pytest.mark.asyncio async def test_execute_write_post_rejects_read_only_sql(): ds = Datasette(memory=True, default_deny=True) From 66bbbbc947bd4d7305761a627dc2f1949949c0a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:35:09 -0700 Subject: [PATCH 1751/1866] Support multi-line parameters on /db/-/execute-write Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536317049 Each paramater input now has an expand/collapse button toggle to turn into a textarea. If you paste text that includes at least one newline it toggles automatically. --- datasette/templates/execute_write.html | 94 +++++++++++++++++++++++++- tests/test_queries.py | 1 + 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 705181d8..a560e920 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -74,6 +74,25 @@ color: #b00020; font-weight: 700; } +form.sql .execute-write-parameter-row textarea[data-parameter-control] { + border: 1px solid #ccc; + border-radius: 3px; + box-sizing: content-box; + display: inline-block; + font-family: Helvetica, sans-serif; + font-size: 1em; + min-height: 7rem; + padding: 9px 4px; + vertical-align: top; + width: 60%; +} +form.sql.core button.execute-write-parameter-toggle[type=button] { + font-size: 0.72rem; + height: 1.8rem; + line-height: 1; + margin-left: 0.35rem; + padding: 0.25rem 0.45rem; +} {% endblock %} @@ -118,7 +137,7 @@ {% if parameter_names %}

      Parameters

      {% for parameter in parameter_names %} -

      +

      {% endfor %} {% endif %} @@ -164,6 +183,79 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} + + {% if write_template_tables %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2b3920f7..e4eaee30 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -680,6 +680,39 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _execute_write_analysis_data(datasette, db, sql, actor): + parameter_names = [] + analysis_rows = [] + analysis_error = None + if sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1024,6 +1057,45 @@ class ExecuteWriteView(BaseView): ) +class ExecuteWriteAnalyzeView(BaseView): + name = "execute-write-analyze" + has_json_alternate = False + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + + try: + data, _ = await _json_or_form_payload(request) + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + if not isinstance(data, dict): + return _block_framing(_error(["JSON must be a dictionary"], 400)) + invalid_keys = set(data) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = data.get("sql") or "" + if not isinstance(sql, str): + return _block_framing(_error(["sql must be a string"], 400)) + return _block_framing( + Response.json( + await _execute_write_analysis_data(self.ds, db, sql, request.actor) + ) + ) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index f44a39fe..2f581661 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -528,6 +528,7 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _ExecuteWriteView: +.. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ @@ -536,6 +537,8 @@ Executing write SQL ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. +``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. + .. _QueryDefinitionView: Getting a saved query definition diff --git a/tests/test_queries.py b/tests/test_queries.py index a6080958..6d2c0b25 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -719,7 +719,9 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text + assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text + assert "refreshExecuteWriteAnalysis" in response.text assert '
      Required permissioninsertupdateread
      ' in response.text assert '' in response.text assert "" in response.text @@ -737,6 +739,53 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text +@pytest.mark.asyncio +async def test_execute_write_analyze_endpoint_uses_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_analyze", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "insert into dogs (name) values (:name)"}, + ) + read_only_response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "select * from dogs where name = :name"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["parameters"] == ["name"] + assert data["analysis_error"] is None + assert data["execute_disabled"] is False + assert data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row", + "source": None, + "allowed": True, + } + ] + assert "params" not in data + + assert read_only_response.status_code == 200 + read_only_data = read_only_response.json() + assert read_only_data["ok"] is False + assert read_only_data["parameters"] == ["name"] + assert read_only_data["analysis_error"] == ( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + assert read_only_data["execute_disabled"] is True + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From de55a76d402a6326c60a5f4cd1a03c7476613f0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:33:57 -0700 Subject: [PATCH 1754/1866] Fix 500 error when accessing query page without ?sql= parameter (#2744) Closes #2743 --- datasette/templates/query.html | 4 ++-- datasette/views/database.py | 43 ++++++++++++++++++---------------- docs/changelog.rst | 7 ++++++ tests/plugins/my_plugin.py | 4 ++-- tests/test_html.py | 16 +++++++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8b405da5..5f85ac6b 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -46,14 +46,14 @@ {% if not hide_sql %} {% if editable and allow_execute_sql %}

      + >{% if query and query.sql %}{{ query.sql }}{% elif tables %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}

      {% else %} 
      {% if query %}{{ query.sql }}{% endif %}
      {% endif %} {% else %} {% if not canned_query %} {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0cf93832..8e4ea85a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -577,7 +577,7 @@ class QueryView(View): named_parameters = [] if canned_query and canned_query.get("params"): named_parameters = canned_query["params"] - if not named_parameters: + if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { named_parameter: params.get(named_parameter) or "" @@ -602,7 +602,7 @@ class QueryView(View): params_for_query = params - if not canned_query_write: + if sql and not canned_query_write: try: if not canned_query: # For regular queries we only allow SELECT, plus other rules @@ -646,6 +646,8 @@ class QueryView(View): # Handle formats from plugins if format_ == "csv": + if not sql: + raise DatasetteError("?sql= is required", status=400) async def fetch_data_for_csv(request, _next=None): results = await db.execute(sql, params, truncate=True) @@ -771,25 +773,26 @@ class QueryView(View): # - No magic parameters, so no :_ in the SQL string edit_sql_url = None is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - datasette.urls.database(database) - + "/-/query" - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } + if sql: + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "/-/query" + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) ) - ) async def query_actions(): query_actions = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index 329b4769..dfb2a736 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_unreleased: + +Unreleased +---------- + +- Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) + .. _v1_0_a30: 1.0a30 (2026-05-24) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 4e401c07..f682e8b9 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -387,8 +387,8 @@ def view_actions(datasette, database, view, actor): @hookimpl def query_actions(datasette, database, query_name, sql): - # Don't explain an explain - if sql.lower().startswith("explain"): + # Don't explain an explain (or a missing query) + if not sql or sql.lower().startswith("explain"): return return [ { diff --git a/tests/test_html.py b/tests/test_html.py index efc1040d..d20796c9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -241,6 +241,22 @@ def test_query_page_truncates(): ] +@pytest.mark.asyncio +async def test_query_page_with_no_sql(ds_client): + # https://github.com/simonw/datasette/issues/2743 + response = await ds_client.get("/fixtures/-/query") + assert response.status_code == 200 + assert '

      +

      + {% set parameter_names = [] %} + {% set parameter_values = {} %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

      @@ -90,5 +95,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5037d006..9b522f66 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -75,61 +75,8 @@ color: #b00020; font-weight: 700; } -form.sql .execute-write-parameter-row textarea[data-parameter-control] { - border: 1px solid #ccc; - border-radius: 3px; - box-sizing: border-box; - display: block; - font-family: Helvetica, sans-serif; - font-size: 1em; - min-height: 7rem; - padding: 9px 4px; - width: 100%; -} -form.sql .execute-write-parameter-row { - align-items: start; - column-gap: 0.6rem; - display: grid; - grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto; - margin: 0 0 0.65rem; - max-width: 52rem; -} -form.sql .execute-write-parameter-row label { - overflow-wrap: anywhere; - padding-top: 0.55rem; - width: auto; -} -form.sql .execute-write-parameter-row input[data-parameter-control] { - box-sizing: border-box; - width: 100%; -} -form.sql.core button.execute-write-parameter-toggle[type=button] { - font-size: 0.72rem; - height: 1.8rem; - line-height: 1; - margin: 0.25rem 0 0; - padding: 0.25rem 0.45rem; -} -@media (max-width: 480px) { - form.sql .execute-write-parameter-row { - grid-template-columns: 1fr; - row-gap: 0.25rem; - } - form.sql .execute-write-parameter-row label { - padding-top: 0; - } - form.sql.core button.execute-write-parameter-toggle[type=button] { - justify-self: start; - margin-top: 0; - } -} -form.sql .execute-write-editor { - max-width: 52rem; -} -form.sql .execute-write-editor textarea#sql-editor { - width: 100%; -} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -168,16 +115,11 @@ form.sql .execute-write-editor textarea#sql-editor { {% endif %} -

      +

      -
      - {% if parameter_names %} -

      Parameters

      - {% for parameter in parameter_names %} -

      - {% endfor %} - {% endif %} -
      + {% set sql_parameters_section_id = "execute-write-parameters-section" %} + {% set sql_parameters_allow_expand = true %} + {% include "_sql_parameters.html" %}

      Query operations

      @@ -222,128 +164,15 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7c251e2c..3bcc7178 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -14,6 +14,7 @@ {% endif %} {% include "_codemirror.html" %} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %} @@ -36,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

      @@ -45,7 +46,7 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

      {% else %} 
      {% if query %}{{ query.sql }}{% endif %}
      @@ -57,12 +58,10 @@ > {% endif %} {% endif %} - {% if named_parameter_values %} -

      Query parameters

      - {% for name, value in named_parameter_values.items() %} -

      - {% endfor %} - {% endif %} + {% set parameter_names = named_parameter_values.keys()|list %} + {% set parameter_values = named_parameter_values %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

      {% if not hide_sql %}{% endif %} @@ -97,5 +96,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index e4eaee30..278f7e8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1061,7 +1061,7 @@ class ExecuteWriteAnalyzeView(BaseView): name = "execute-write-analyze" has_json_alternate = False - async def post(self, request): + async def get(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( action="execute-write-sql", @@ -1072,13 +1072,7 @@ class ExecuteWriteAnalyzeView(BaseView): _error(["Permission denied: need execute-write-sql"], 403) ) - try: - data, _ = await _json_or_form_payload(request) - except QueryValidationError as ex: - return _block_framing(_error([ex.message], ex.status)) - if not isinstance(data, dict): - return _block_framing(_error(["JSON must be a dictionary"], 400)) - invalid_keys = set(data) - {"sql"} + invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( _error( @@ -1086,9 +1080,7 @@ class ExecuteWriteAnalyzeView(BaseView): 400, ) ) - sql = data.get("sql") or "" - if not isinstance(sql, str): - return _block_framing(_error(["sql must be a string"], 400)) + sql = request.args.get("sql") or "" return _block_framing( Response.json( await _execute_write_analysis_data(self.ds, db, sql, request.actor) @@ -1096,6 +1088,34 @@ class ExecuteWriteAnalyzeView(BaseView): ) +class QueryParametersView(BaseView): + name = "query-parameters" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + try: + parameters = _derived_query_parameters(request.args.get("sql") or "") + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.json({"ok": True, "parameters": parameters})) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index 2f581661..91ed5306 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -527,17 +527,20 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +.. _QueryParametersView: .. _ExecuteWriteView: .. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ +``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. + ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index a9d22036..ae2c74e0 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -200,7 +200,10 @@ def test_error_in_on_success_message_sql(canned_write_client): def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") - assert '' in response.text + assert ( + '' + in response.text + ) def test_canned_query_pages_no_vary_header(canned_write_client): diff --git a/tests/test_html.py b/tests/test_html.py index e5f00e17..b49391a6 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -326,17 +326,29 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) +@pytest.mark.asyncio +async def test_database_page_sql_parameter_refresh_markup(ds_client): + response = await ds_client.get("/fixtures") + assert response.status_code == 200 + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text + + @pytest.mark.asyncio async def test_row_html_simple_primary_key(ds_client): response = await ds_client.get("/fixtures/simple_primary_key/1") diff --git a/tests/test_queries.py b/tests/test_queries.py index 6d2c0b25..23820cf3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -721,7 +721,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="delete"' in response.text assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text - assert "refreshExecuteWriteAnalysis" in response.text + assert "setupSqlParameterRefresh" in response.text assert '

      Required permissioninsert
      ' in response.text assert '' in response.text assert "" in response.text @@ -747,15 +747,15 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() - response = await ds.client.post( + response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "insert into dogs (name) values (:name)"}, + params={"sql": "insert into dogs (name) values (:name)"}, ) - read_only_response = await ds.client.post( + read_only_response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "select * from dogs where name = :name"}, + params={"sql": "select * from dogs where name = :name"}, ) assert response.status_code == 200 @@ -786,6 +786,44 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): assert read_only_data["execute_disabled"] is True +@pytest.mark.asyncio +async def test_query_parameters_endpoint_uses_get_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_parameters", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={ + "sql": "select * from dogs where name = :name and id = :id", + }, + ) + permission_denied_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "not-root"}, + params={"sql": "select * from dogs where name = :name"}, + ) + magic_parameter_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={"sql": "select :_actor_id"}, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "parameters": ["name", "id"]} + assert permission_denied_response.status_code == 403 + assert permission_denied_response.json()["errors"] == [ + "Permission denied: need execute-sql" + ] + assert magic_parameter_response.status_code == 400 + assert magic_parameter_response.json()["errors"] == [ + "Magic parameters are not allowed" + ] + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From 4208ded249b28f8b0918ce80d289bfc88f9e8921 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:46:21 -0700 Subject: [PATCH 1756/1866] No execute-write on immutable databases Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536690161 --- datasette/default_database_actions.py | 2 ++ datasette/views/database.py | 7 ++++ tests/test_queries.py | 46 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py index 78055392..e0cb3cdf 100644 --- a/datasette/default_database_actions.py +++ b/datasette/default_database_actions.py @@ -5,6 +5,8 @@ from datasette.resources import DatabaseResource @hookimpl def database_actions(datasette, actor, database, request): async def inner(): + if not datasette.get_database(database).is_mutable: + return [] if not await datasette.allowed( action="execute-write-sql", resource=DatabaseResource(database), diff --git a/datasette/views/database.py b/datasette/views/database.py index 278f7e8c..de02cd0f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -964,6 +964,13 @@ class ExecuteWriteView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ) + if not db.is_mutable: + return _block_framing( + _error( + ["Cannot execute write SQL because this database is immutable."], + 403, + ) + ) return await self._render_form( request, db, diff --git a/tests/test_queries.py b/tests/test_queries.py index 23820cf3..c31d7205 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -858,6 +858,52 @@ async def test_database_action_menu_links_to_execute_write_for_permitted_actor() assert "Execute write SQL" in writer_response.text +@pytest.mark.asyncio +async def test_database_action_menu_hides_execute_write_for_immutable_database(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_menu_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get("/data", actor={"id": "writer"}) + + assert response.status_code == 200 + assert "Execute write SQL" not in response.text + assert 'href="/data/-/execute-write"' not in response.text + + +@pytest.mark.asyncio +async def test_execute_write_get_rejects_immutable_database(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_get_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Cannot execute write SQL because this database is immutable." + ] + + @pytest.mark.asyncio async def test_execute_write_post_requires_database_and_table_permissions(): ds = Datasette( From 8ab8999ba97e0ec1d113ee8d3954d6431f39fa28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:55:36 -0700 Subject: [PATCH 1757/1866] Big visual improvement to /-/queries pages Including /db/-/queries Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4536860239 --- datasette/templates/query_list.html | 226 ++++++++++++++++++++++++---- datasette/views/database.py | 12 +- tests/test_queries.py | 25 ++- 3 files changed, 229 insertions(+), 34 deletions(-) diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index af974550..dbd607ab 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -2,6 +2,155 @@ {% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} +{% block extra_head %} +{{- super() -}} + +{% endblock %} + {% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} @@ -10,49 +159,66 @@ {% block content %} -

      Queries

      +
      - -

      +

      Queries

      + + +

      -

      - - - - -

      +
      +
      + Mode + + + +
      +
      + Publication + + + +
      +
      {% if queries %} -
        - {% for query in queries %} -
      • - {% if show_database %} - {{ query.database }}: - {% endif %} - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} - {% if query.is_write %}Writable{% endif %} - {% if query.is_published %}Published{% endif %} -
        • - {% endfor %} -
      +
      Required permissioninsert
      + + + {% if show_database %}{% endif %} + + + + + + + {% for query in queries %} + + {% if show_database %} + + {% endif %} + + + + + {% endfor %} + +
      DatabaseQueryModePublication
      {{ query.database }} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.description %}

      {{ query.description }}

      {% endif %} +       		{% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}
      {% else %}

      No queries found.

      {% endif %} {% if next_url %} -

      Next page

      + {% endif %} +
    + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index de02cd0f..3c660bc7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -487,9 +487,9 @@ def _as_optional_bool(value, name): raise QueryValidationError("{} must be 0 or 1".format(name)) -def _query_list_limit(value): +def _query_list_limit(value, default=50): if value in (None, ""): - return 50 + return default try: return min(max(1, int(value)), 1000) except ValueError as ex: @@ -1136,7 +1136,10 @@ class QueryListView(BaseView): database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: - limit = _query_list_limit(request.args.get("_size")) + limit = _query_list_limit( + request.args.get("_size"), + default=20 if format_ == "html" else 50, + ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_published = _as_optional_bool( request.args.get("is_published"), "is_published" @@ -1175,6 +1178,9 @@ class QueryListView(BaseView): data = { "ok": True, "database": database, + "database_color": ( + self.ds.get_database(database).color if database is not None else None + ), "queries": page["queries"], "next": page["next"], "next_url": next_url, diff --git a/tests/test_queries.py b/tests/test_queries.py index c31d7205..b7416ac7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -451,12 +451,34 @@ async def test_query_list_search_filter_and_html(): assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text + assert 'class="query-list-results"' in html_response.text + assert "Mode" in html_response.text + assert 'type="radio" name="is_published" value="1"' in html_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] +@pytest.mark.asyncio +async def test_query_list_html_defaults_to_twenty_and_shows_pagination(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html_pagination", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + response = await ds.client.get("/data/-/queries", actor={"id": "root"}) + json_response = await ds.client.get("/data/-/queries.json", actor={"id": "root"}) + + assert response.status_code == 200 + assert response.text.count('aria-label="Query pagination"') == 1 + assert "Demo query 20" in response.text + assert "Demo query 21" not in response.text + assert 'href="/data/-/queries?_next=' in response.text + assert len(json_response.json()["queries"]) == 25 + + @pytest.mark.asyncio async def test_global_query_list_api_and_html(): ds = Datasette(memory=True) @@ -519,7 +541,8 @@ async def test_global_query_list_api_and_html(): ("beta", "beta_first"), ] assert html_response.status_code == 200 - assert 'href="/beta">beta:' in html_response.text + assert 'Database' in html_response.text + assert 'class="query-list-database" href="/beta">beta' in html_response.text assert "Beta first" in html_response.text assert "Alpha first" not in html_response.text From f1dd86ebfb01644fead19f9f007b9b76f863d72e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 14:05:26 -0700 Subject: [PATCH 1758/1866] Tweak URL designs of new endpoints --- datasette/app.py | 6 +++--- datasette/templates/database.html | 2 +- datasette/templates/execute_write.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/query_create.html | 2 +- docs/json_api.rst | 6 +++--- queries-plan.md | 4 ++-- tests/test_html.py | 4 ++-- tests/test_queries.py | 22 +++++++++++----------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90e41521..232aa0cf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2745,11 +2745,11 @@ class Datasette: ) add_route( QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/insert$", + r"/(?P[^\/\.]+)/-/queries/insert$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/-/analyze$", + r"/(?P[^\/\.]+)/-/execute-write/analyze$", ) add_route( ExecuteWriteView.as_view(self), @@ -2761,7 +2761,7 @@ class Datasette: ) add_route( QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/-/parameters$", + r"/(?P[^\/\.]+)/-/query/parameters$", ) add_route( wrap_view(QueryView, self), diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 0c9ec94c..62f9c620 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -26,7 +26,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} -
    +

    Custom SQL query

    {% set parameter_names = [] %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 9b522f66..46f58c3b 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -95,7 +95,7 @@

    {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

    {% endif %} - + {% if write_template_tables %}
    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 3bcc7178..f74d21f1 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -37,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index fb2599d2..3c027def 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -17,7 +17,7 @@

    Create query

    - +


    diff --git a/docs/json_api.rst b/docs/json_api.rst index 91ed5306..dd54c459 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -525,7 +525,7 @@ Creating saved queries in the UI Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: @@ -534,13 +534,13 @@ Creating saved queries Executing write SQL ~~~~~~~~~~~~~~~~~~~ -``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. +``GET //-/query/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/queries-plan.md b/queries-plan.md index a708e887..72427df2 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -211,7 +211,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: - `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/-/insert` creates a query. +- `POST /{database}/-/queries/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. - `POST /{database}/{query}/-/delete` deletes one query. @@ -388,7 +388,7 @@ The read methods should reconstruct the existing dictionary shape used by query On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/-/insert` and default to `is_published=false`. +The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. diff --git a/tests/test_html.py b/tests/test_html.py index b49391a6..8cda6dba 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -329,7 +329,7 @@ async def test_query_parameter_form_fields(ds_client): ' ' in response.text ) - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") @@ -344,7 +344,7 @@ async def test_query_parameter_form_fields(ds_client): async def test_database_page_sql_parameter_refresh_markup(ds_client): response = await ds_client.get("/fixtures") assert response.status_code == 200 - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text diff --git a/tests/test_queries.py b/tests/test_queries.py index b7416ac7..57920584 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -356,7 +356,7 @@ async def test_query_insert_api_creates_read_only_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -568,7 +568,7 @@ async def test_query_insert_api_publish_requires_publish_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "writer"}, json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, ) @@ -586,7 +586,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -603,7 +603,7 @@ async def test_query_insert_api_creates_writable_query(): assert query["parameters"] == ["name"] bad_response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -671,7 +671,7 @@ async def test_query_insert_api_rejects_magic_parameters(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -742,7 +742,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text - assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text + assert 'data-analyze-url="/data/-/execute-write/analyze"' in response.text assert 'addEventListener("paste"' in response.text assert "setupSqlParameterRefresh" in response.text assert '' in response.text @@ -771,12 +771,12 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "insert into dogs (name) values (:name)"}, ) read_only_response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "select * from dogs where name = :name"}, ) @@ -818,19 +818,19 @@ async def test_query_parameters_endpoint_uses_get_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={ "sql": "select * from dogs where name = :name and id = :id", }, ) permission_denied_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "not-root"}, params={"sql": "select * from dogs where name = :name"}, ) magic_parameter_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={"sql": "select :_actor_id"}, ) From 4a1a4d7807fb99203b9053b6d270b265df61f0af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 11:59:49 -0700 Subject: [PATCH 1759/1866] Query is_trusted and is_private properties Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4547270516 Diff explanation: https://gist.github.com/simonw/1e4de6c4b041a51968eb273ee96dec1f --- datasette/app.py | 39 ++-- datasette/default_actions.py | 7 - datasette/default_permissions/defaults.py | 100 +++++---- datasette/templates/query_create.html | 4 +- datasette/templates/query_list.html | 65 +++++- datasette/utils/internal_db.py | 3 +- datasette/views/database.py | 79 ++++--- docs/authentication.rst | 10 - docs/internals.rst | 3 +- queries-plan.md | 84 ++++---- tests/test_queries.py | 245 ++++++++++++++++++---- 11 files changed, 421 insertions(+), 218 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 232aa0cf..3329ee7e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,7 +618,8 @@ class Datasette: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - is_published=bool(query_config.get("is_published")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), source="config", on_success_message=query_config.get("on_success_message"), on_success_message_sql=query_config.get("on_success_message_sql"), @@ -1084,7 +1085,8 @@ class Datasette: "parameters": parameters, "is_write": is_write, "write": is_write, - "is_published": bool(row["is_published"]), + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), "source": row["source"], "owner_id": row["owner_id"], "on_success_message": options.get("on_success_message"), @@ -1119,7 +1121,8 @@ class Datasette: fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -1144,8 +1147,8 @@ class Datasette: sql_statement = """ INSERT INTO queries ( database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_published, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ if replace: sql_statement += """ @@ -1157,7 +1160,8 @@ class Datasette: options = excluded.options, parameters = excluded.parameters, is_write = excluded.is_write, - is_published = excluded.is_published, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, source = excluded.source, owner_id = excluded.owner_id, updated_at = CURRENT_TIMESTAMP @@ -1174,7 +1178,8 @@ class Datasette: options_json, parameters_json, int(bool(is_write)), - int(bool(is_published)), + int(bool(is_private)), + int(bool(is_trusted)), source, owner_id, ], @@ -1193,7 +1198,8 @@ class Datasette: fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -1209,7 +1215,8 @@ class Datasette: "description_html": description_html, "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": is_private, + "is_trusted": is_trusted, "source": source, "owner_id": owner_id, } @@ -1227,7 +1234,7 @@ class Datasette: for field, value in fields.items(): if value is UNCHANGED: continue - if field in {"is_write", "is_published"}: + if field in {"is_write", "is_private", "is_trusted"}: value = int(bool(value)) elif field == "parameters": value = json.dumps(list(value or [])) @@ -1300,7 +1307,8 @@ class Datasette: cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, include_private=False, @@ -1372,9 +1380,12 @@ class Datasette: if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) - if is_published is not None: - where_clauses.append("q.is_published = :query_is_published") - params["query_is_published"] = int(bool(is_published)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) if source is not None: where_clauses.append("q.source = :query_source") params["query_source"] = source diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6787b80e..6a1f77b8 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -68,13 +68,6 @@ def register_actions(): resource_class=DatabaseResource, also_requires="execute-sql", ), - Action( - name="publish-query", - abbr="pq", - description="Publish saved queries for actors without execute-sql", - resource_class=DatabaseResource, - also_requires="insert-query", - ), # Table-level actions (child-level) Action( name="view-table", diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 58deea01..dfd8d3e9 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -26,6 +26,32 @@ DEFAULT_ALLOW_ACTIONS = frozenset( ) +def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: + selects = [] + params = {} + for index, (database_name, db_config) in enumerate( + ((datasette.config or {}).get("databases") or {}).items() + ): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if isinstance(query_config, dict) and query_config.get("is_private"): + continue + parent_param = f"query_config_parent_{index}_{len(selects)}" + child_param = f"query_config_child_{index}_{len(selects)}" + selects.append( + f""" + SELECT :{parent_param} AS parent, :{child_param} AS child + WHERE NOT EXISTS ( + SELECT 1 FROM queries + WHERE database_name = :{parent_param} + AND name = :{child_param} + ) + """ + ) + params[parent_param] = database_name + params[child_param] = query_name + return selects, params + + @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -93,61 +119,45 @@ async def default_query_permissions_sql( if action != "view-query": return None - execute_sql = await datasette.allowed_resources_sql( - action="execute-sql", actor=actor - ) - sql = execute_sql.sql - params = {} - for key, value in execute_sql.params.items(): - new_key = f"query_execute_sql_{key}" - sql = sql.replace(f":{key}", f":{new_key}") - params[new_key] = value - - trusted_writable_sql = "" + params = {"query_owner_id": actor_id} + rule_sqls = [] if not datasette.default_deny: - trusted_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, - 'trusted writable query' AS reason + 'non-private query' AS reason FROM queries - WHERE is_write = 1 - AND source IN ('config', 'plugin') - """ + WHERE is_private = 0 + """ + ) - user_writable_sql = "" if actor_id is not None: - params["query_owner_id"] = actor_id - user_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries - WHERE is_write = 1 - AND source = 'user' - AND owner_id = :query_owner_id + WHERE owner_id = :query_owner_id + """ + ) + + config_restriction_selects, config_restriction_params = ( + _configured_query_restriction_selects(datasette) + ) + + restriction_sqls = [ """ + SELECT database_name AS parent, name AS child + FROM queries + WHERE is_private = 0 + OR owner_id = :query_owner_id + """ + ] + restriction_sqls.extend(config_restriction_selects) + params.update(config_restriction_params) return PermissionSQL( - sql=f""" - WITH execute_sql_allowed AS ( - {sql} - ) - SELECT database_name AS parent, name AS child, 1 AS allow, - 'published query' AS reason - FROM queries - WHERE is_write = 0 - AND is_published = 1 - UNION ALL - SELECT q.database_name AS parent, q.name AS child, 1 AS allow, - 'execute-sql allows query' AS reason - FROM queries q - JOIN execute_sql_allowed es - ON es.parent = q.database_name - AND es.child IS NULL - WHERE q.is_write = 0 - AND q.is_published = 0 - {trusted_writable_sql} - {user_writable_sql} - """, + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql="\nUNION ALL\n".join(restriction_sqls), params=params, ) diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 3c027def..686d971e 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -27,9 +27,7 @@

    - {% if can_publish %} -

    - {% endif %} +

    {% if sql and analysis_is_write %}

    Execute write SQL

    {% endif %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index dbd607ab..25259b3d 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -73,7 +73,7 @@ border-collapse: collapse; font-size: 0.9rem; margin: 0.25rem 0 1rem; - min-width: 36rem; + min-width: 42rem; width: 100%; } .query-list-results th, @@ -100,6 +100,16 @@ font-size: 0.78rem; margin: 0.15rem 0 0; } +.query-list-owner { + color: #39445a; + font-family: var(--font-monospace, monospace); + white-space: nowrap; +} +.query-list-flags { + display: flex; + flex-wrap: wrap; + gap: 0.3rem; +} .query-list-pill { background-color: #eef1f5; border: 1px solid #d7dde5; @@ -116,15 +126,36 @@ background-color: #fff4db; border-color: #e2b64e; } -.query-list-pill-published { +.query-list-pill-public { background-color: #e7f5ec; border-color: #9ecfab; color: #267a3e; } -.query-list-pill-unpublished { +.query-list-pill-private { background-color: #f7edf0; border-color: #dbb8c1; } +.query-list-pill-trusted { + background-color: #e7f5ec; + border-color: #9ecfab; + color: #267a3e; +} +.query-list-empty { + color: #6b7280; +} +.query-list-footnotes { + border-top: 1px solid #d7dde5; + color: #4f5b6d; + font-size: 0.82rem; + margin: 0.35rem 0 1rem; + padding-top: 0.55rem; +} +.query-list-footnotes p { + margin: 0.25rem 0; +} +.query-list-footnotes .query-list-pill { + margin-right: 0.35rem; +} .query-list-pagination a { border: 1px solid #007bff; border-radius: 0.25rem; @@ -177,10 +208,10 @@
    - Publication - - - + Visibility + + +
    @@ -191,8 +222,8 @@     {% if show_database %}{% endif %} - - + + @@ -205,12 +236,24 @@ {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.description %}

    {{ query.description }}

    {% endif %} - - + + {% endfor %}
    DatabaseQueryModePublicationOwnerFlags
    {% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}{% if query.owner_id is not none %}{{ query.owner_id }}{% else %}-{% endif %} + + {% if query.is_write %}Writable{% else %}Read-only{% endif %} + {% if query.is_private %}Private{% endif %} + {% if query.is_trusted %}Trusted{% endif %} + +
    + {% if show_private_note or show_trusted_note %} +
    + {% if show_private_note %}

    PrivateOnly the owning actor can view this query.

    {% endif %} + {% if show_trusted_note %}

    TrustedExecution skips the usual SQL and write permission checks after view-query allows access.

    {% endif %} +
    + {% endif %} {% else %}

    No queries found.

    {% endif %} diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 9c693b0a..bf172667 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -123,7 +123,8 @@ async def initialize_metadata_tables(db): options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/datasette/views/database.py b/datasette/views/database.py index 3c660bc7..91e9c350 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,7 @@ _query_fields = { "fragment", "parameters", "params", - "is_published", + "is_private", "on_success_message", "on_success_message_sql", "on_success_redirect", @@ -571,7 +571,7 @@ async def _check_query_name(db, name, *, existing=False): raise QueryValidationError("Query name conflicts with a table or view") -async def _analyze_user_query(datasette, db, sql, *, actor, is_published): +async def _analyze_user_query(datasette, db, sql, *, actor): if not sql or not isinstance(sql, str): raise QueryValidationError("SQL is required") derived = _derived_query_parameters(sql) @@ -583,8 +583,6 @@ async def _analyze_user_query(datasette, db, sql, *, actor, is_published): is_write = _analysis_is_write(analysis) if is_write: - if is_published: - raise QueryValidationError("Writable queries cannot be published") try: await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis @@ -680,6 +678,26 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): + if query.get("is_trusted"): + return + if query.get("write"): + await datasette.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + await datasette.ensure_query_write_permissions( + db.name, query["sql"], actor=actor + ) + else: + await datasette.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + + async def _execute_write_analysis_data(datasette, db, sql, actor): parameter_names = [] analysis_rows = [] @@ -752,7 +770,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): def _apply_query_data_types(data): typed = dict(data) - for key in ("hide_sql", "is_published"): + for key in ("hide_sql", "is_private"): if key in typed: typed[key] = _as_bool(typed[key]) return typed @@ -769,20 +787,12 @@ async def _prepare_query_create(datasette, request, db, data): if await datasette.get_query(db.name, name) is not None: raise QueryValidationError("Query already exists") - is_published = _as_bool(data.get("is_published")) is_write, derived, analysis = await _analyze_user_query( datasette, db, data.get("sql"), actor=request.actor, - is_published=is_published, ) - if is_published and not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError("Permission denied: need publish-query", status=403) if not is_write and any(data.get(field) for field in _query_write_fields): raise QueryValidationError("Writable query fields require writable SQL") @@ -800,7 +810,8 @@ async def _prepare_query_create(datasette, request, db, data): "fragment": data.get("fragment"), "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": _as_bool(data.get("is_private", True)), + "is_trusted": False, "source": "user", "owner_id": _actor_id(request.actor), "on_success_message": data.get("on_success_message"), @@ -819,7 +830,6 @@ async def _prepare_query_update(datasette, request, db, existing, update): update = _apply_query_data_types(update) sql = update.get("sql", existing["sql"]) - is_published = update.get("is_published", existing["is_published"]) query_is_write = existing["is_write"] derived = _derived_query_parameters(sql) parameters = None @@ -830,19 +840,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): db, sql, actor=request.actor, - is_published=is_published, ) - elif is_published and query_is_write: - raise QueryValidationError("Writable queries cannot be published") - if is_published and not existing["is_published"]: - if not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError( - "Permission denied: need publish-query", status=403 - ) if "parameters" in update or "params" in update: parameters = _coerce_query_parameters( @@ -864,7 +862,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): "fragment": update.get("fragment"), "parameters": parameters, "is_write": query_is_write, - "is_published": is_published, + "is_private": update.get("is_private"), "on_success_message": update.get("on_success_message"), "on_success_message_sql": update.get("on_success_message_sql"), "on_success_redirect": update.get("on_success_redirect"), @@ -1141,8 +1139,8 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_published = _as_optional_bool( - request.args.get("is_published"), "is_published" + is_private = _as_optional_bool( + request.args.get("is_private"), "is_private" ) except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1154,7 +1152,7 @@ class QueryListView(BaseView): cursor=request.args.get("_next"), q=request.args.get("q") or None, is_write=is_write, - is_published=is_published, + is_private=is_private, source=request.args.get("source") or None, owner_id=request.args.get("owner_id") or None, include_private=True, @@ -1186,12 +1184,14 @@ class QueryListView(BaseView): "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "show_private_note": any(query["is_private"] for query in page["queries"]), + "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", - "is_published": request.args.get("is_published") or "", + "is_private": request.args.get("is_private") or "", "source": request.args.get("source") or "", "owner_id": request.args.get("owner_id") or "", }, @@ -1255,11 +1255,6 @@ class QueryCreateView(BaseView): "database_color": db.color, "sql": sql, "parameter_names": parameter_names, - "can_publish": await self.ds.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ), "analysis_error": analysis_error, "analysis_rows": analysis_rows, "analysis_is_write": bool( @@ -1435,9 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write") and canned_query.get("source") == "user": - await datasette.ensure_query_write_permissions( - db.name, canned_query["sql"], actor=request.actor + if canned_query.get("write"): + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor ) # If database is immutable, return an error @@ -1558,6 +1553,10 @@ class QueryView(View): ) if not visible: raise Forbidden("You do not have permission to view this query") + if not canned_query_write: + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) else: await datasette.ensure_permission( diff --git a/docs/authentication.rst b/docs/authentication.rst index b6a4cb7e..6e835c8d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1299,16 +1299,6 @@ insert-query Actor is allowed to create saved queries in a database. -``resource`` - ``datasette.resources.DatabaseResource(database)`` - ``database`` is the name of the database (string) - -.. _actions_publish_query: - -publish-query -------------- - -Actor is allowed to publish a saved read-only query so actors without ``execute-sql`` can run it. - ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/internals.rst b/docs/internals.rst index b5da7cbf..c76de487 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2158,7 +2158,8 @@ The internal database schema is as follows: options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/queries-plan.md b/queries-plan.md index 72427df2..f4b8049c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -13,9 +13,9 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Internal table name: `queries`. - Query definitions should use real columns, not a JSON blob for all options. - Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No `queries_database_is_published_idx` index. -- User-created queries require `execute-sql` and `insert-query` on the database. Writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- `publish-query` is the permission for creating or updating a query so users without `execute-sql` can execute it. +- No separate index is needed for the privacy/trust flags yet. +- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. +- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. - Add `update-query` and `delete-query`, so administrators can manage queries created by other users. - Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. - Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. @@ -45,7 +45,8 @@ CREATE TABLE IF NOT EXISTS queries ( options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -64,11 +65,12 @@ Column notes: - Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. - `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. - Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. +- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. +- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. - `source` distinguishes `user`, `config`, and `plugin` rows. - `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. -No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_is_published_idx` index for now. +No separate index is needed on `(database_name, name)` because the primary key already creates one. `QueryResource.resources_sql()` can become: @@ -104,7 +106,6 @@ Remove the old `canned_queries()` hookspec and all core calls to it. If compatib Add core actions: - `insert-query`, database-level, for creating queries in a database. -- `publish-query`, database-level, for marking read-only queries as executable by actors who lack `execute-sql`. - `update-query`, query-level, for modifying existing query definitions. - `delete-query`, query-level, for deleting existing query definitions. @@ -114,17 +115,11 @@ User-created query creation requires: - `insert-query` on `DatabaseResource(database)` - If analysis shows the query is writable, the table-level write permissions described in the writable query section. -Setting `is_published=1` requires: - -- `publish-query` on `DatabaseResource(database)` -- The query must be read-only according to `Database.analyze_sql()`. - Updating an existing query requires: - `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - If the SQL changes, also require `execute-sql` on the database. - If the changed SQL is writable, also require the table-level write permissions described in the writable query section. -- If `is_published` changes from `0` to `1`, also require `publish-query` on the database. Deleting an existing query requires: @@ -133,18 +128,18 @@ Deleting an existing query requires: Default owner permissions: - For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- Do not automatically grant execution if the user no longer has the execution permission described below. +- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. ## Executing queries Default execution rule for read-only queries: -- If `is_published=0`, the actor needs `execute-sql` on the database. -- If `is_published=1`, the actor can execute the query without `execute-sql`. +- If `is_trusted=0`, the actor needs `execute-sql` on the database. +- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. Default execution rule for user-created writable queries: -- `is_published` must be `0`. +- `is_trusted` must be `0`. - The actor must have `view-query`. - The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. @@ -152,14 +147,14 @@ Implementation: - Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. - Replace it with query-aware default `view-query` permission SQL. -- For `is_published=1 AND is_write=0`, emit a child-level `view-query` allow. -- For `is_published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. -- For `is_write=1 AND source='user'`, emit `view-query` only for the owner or actors with explicit `view-query` permission, then have `QueryView` perform the fresh analysis/table-permission check before execution. -- For trusted writable queries, preserve current behavior by emitting child-level `view-query` allows for `is_write=1 AND source IN ('config', 'plugin')` when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for the owning actor. +- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. +- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. -For read-only queries this keeps `QueryView` simple: it checks `view-query` for the query resource, and the default permission hook encodes the relationship with `execute-sql`. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. +For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. -Explicit deny rules should still be able to block a published query. +Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. ## Writable queries @@ -180,7 +175,7 @@ Validation flow for user-created queries: 1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. 2. If analysis raises a SQLite error, reject the query. 3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_published=0`. +4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. 5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. 6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - `insert` -> `insert-row` @@ -200,7 +195,7 @@ Fail closed cases for user-created writable queries: - Analysis reports any write operation that cannot be mapped to a Datasette table resource. - Analysis reports writes outside the target database. - The actor lacks any required table write permission. -- `is_published=1` is requested. +- `is_trusted=1` is requested through the user-facing API. This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. @@ -225,7 +220,7 @@ Create request: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, "parameters": ["region"] } } @@ -242,7 +237,8 @@ Successful create returns `201` and the created query definition: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, + "is_trusted": false, "parameters": ["region"] } } @@ -254,7 +250,7 @@ Update request, imitating `RowUpdateView`: { "update": { "title": "Top customers by revenue", - "is_published": true + "is_private": false }, "return": true } @@ -270,7 +266,8 @@ Successful update returns `{"ok": true}` by default. With `"return": true`, retu "name": "top_customers", "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers by revenue", - "is_published": true + "is_private": false, + "is_trusted": false } } ``` @@ -317,7 +314,8 @@ await datasette.add_query( fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -340,7 +338,8 @@ await datasette.update_query( fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -360,7 +359,8 @@ await datasette.list_queries( cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, ) @@ -382,15 +382,13 @@ For column-backed fields, `None` should write SQL `NULL`. For option fields, `No Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. ## Query page save UI On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. - -If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. @@ -403,7 +401,7 @@ This page should require `execute-sql` and `insert-query` to access. It should p - Read-only - Writable -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and optional published status if the actor has `publish-query`. +Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: @@ -413,7 +411,7 @@ Writable mode should always run `Database.analyze_sql()` and show an analysis pa - whether the actor has that permission - source, when the operation comes from a trigger or view -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. Writable mode should not show a publish control, because user-created writable queries cannot be published. +The Save button should be disabled until analysis succeeds and every required table write permission is allowed. The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. @@ -427,14 +425,16 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. - `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. -- Unpublished read-only query requires `execute-sql` to execute. -- Published read-only query can be executed without `execute-sql`. -- Setting `is_published=true` requires `publish-query`. +- Private query is only visible to its owner, even when a broader `view-query` rule applies. +- Non-trusted read-only query requires `execute-sql` to execute. +- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. +- Config queries default to trusted and can opt out with `is_trusted: false`. +- User API rejects client-supplied `is_trusted`. - User-created query requires both `execute-sql` and `insert-query`. - User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. - `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. - User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be published. +- User-created writable query cannot be trusted through the user API. - Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. - Query delete uses `POST /{database}/{query}/-/delete`. - There are no `PATCH` or HTTP `DELETE` routes for query management. diff --git a/tests/test_queries.py b/tests/test_queries.py index 57920584..c97b5733 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,7 +15,6 @@ async def add_numbered_queries(ds, database, count): "select {} as query_number".format(i), title="Demo query {:02d}".format(i), description="Seeded demo query number {:02d}".format(i), - is_published=True, source="user", owner_id="root", ) @@ -44,7 +43,8 @@ async def test_queries_internal_table_schema(): "options", "parameters", "is_write", - "is_published", + "is_private", + "is_trusted", "source", "owner_id", "created_at", @@ -67,7 +67,7 @@ async def test_add_get_and_remove_query(): hide_sql=True, fragment="chart", parameters=["region"], - is_published=True, + is_trusted=True, source="user", owner_id="alice", ) @@ -100,7 +100,8 @@ async def test_add_get_and_remove_query(): "parameters": ["region"], "is_write": False, "write": False, - "is_published": True, + "is_private": False, + "is_trusted": True, "source": "user", "owner_id": "alice", "on_success_message": None, @@ -161,7 +162,8 @@ async def test_update_query_only_updates_provided_fields(): assert query["params"] == [] assert query["on_success_redirect"] is None assert query["sql"] == "select 1" - assert query["is_published"] is False + assert query["is_private"] is False + assert query["is_trusted"] is False options_row = ( await ds.get_internal_database().execute( """ @@ -208,7 +210,8 @@ async def test_config_queries_imported_to_internal_table(): "parameters": ["name"], "is_write": False, "write": False, - "is_published": False, + "is_private": False, + "is_trusted": True, "source": "config", "owner_id": None, "on_success_message": None, @@ -232,30 +235,171 @@ async def test_query_resources_come_from_internal_table(): @pytest.mark.asyncio -async def test_unpublished_query_requires_execute_sql_but_published_does_not(): - ds = Datasette(memory=True, settings={"default_allow_sql": False}) +async def test_default_deny_blocks_view_query_even_for_trusted_query(): + ds = Datasette(memory=True, default_deny=True) ds.add_memory_database("query_permissions", name="data") await ds.invoke_startup() - await ds.add_query("data", "unpublished", "select 1", is_published=False) - await ds.add_query("data", "published", "select 1", is_published=True) + await ds.add_query("data", "trusted", "select 1", is_trusted=True) assert not await ds.allowed( - action="execute-sql", - resource=DatabaseResource("data"), + action="view-query", + resource=QueryResource("data", "trusted"), actor=None, ) + + +@pytest.mark.asyncio +async def test_private_query_restriction_blocks_broad_view_query_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "*"}, + } + } + } + }, + ) + ds.add_memory_database("private_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) assert not await ds.allowed( action="view-query", - resource=QueryResource("data", "unpublished"), - actor=None, + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, ) assert await ds.allowed( action="view-query", - resource=QueryResource("data", "published"), - actor=None, + resource=QueryResource("data", "shared_report"), + actor={"id": "bob"}, ) +@pytest.mark.asyncio +async def test_config_query_restriction_does_not_override_private_internal_query(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("private_query_with_config_name", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + ds.config = { + "databases": { + "data": { + "permissions": {"view-query": {"id": "*"}}, + "queries": {"private_report": {"sql": "select 2"}}, + } + } + } + + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_untrusted_shared_query_execution_requires_execute_sql(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "viewer"}, + "view-query": {"id": "viewer"}, + } + } + } + }, + ) + ds.add_memory_database("untrusted_query_execution", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "shared_report", + "select 1 as one", + is_private=False, + is_trusted=False, + source="user", + owner_id="alice", + ) + + denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert denied.status_code == 403 + + ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} + allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert allowed.status_code == 200 + assert allowed.json()["rows"] == [{"one": 1}] + + +@pytest.mark.asyncio +async def test_config_queries_are_trusted_by_default_but_can_opt_out(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "viewer"}, + }, + "queries": { + "trusted_report": {"sql": "select 1 as one"}, + "untrusted_report": { + "sql": "select 2 as two", + "is_trusted": False, + }, + }, + } + } + }, + ) + ds.add_memory_database("trusted_query_config", name="data") + await ds.invoke_startup() + + trusted = await ds.client.get("/data/trusted_report.json", actor={"id": "viewer"}) + untrusted = await ds.client.get( + "/data/untrusted_report.json", actor={"id": "viewer"} + ) + + assert trusted.status_code == 200 + assert trusted.json()["rows"] == [{"one": 1}] + assert untrusted.status_code == 403 + + @pytest.mark.asyncio async def test_database_page_query_preview_is_limited(): ds = Datasette(memory=True) @@ -281,7 +425,6 @@ async def test_query_actions_are_registered(): assert ds.get_action("execute-write-sql").resource_class is DatabaseResource assert ds.get_action("insert-query").resource_class is DatabaseResource - assert ds.get_action("publish-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -430,21 +573,33 @@ async def test_query_list_search_filter_and_html(): "private_query", "select 'private'", title="Private query", - is_published=False, + is_private=True, source="user", owner_id="root", ) + await ds.add_query( + "data", + "trusted_query", + "select 'trusted'", + title="Trusted query", + is_trusted=True, + source="config", + ) html_response = await ds.client.get( "/data/-/queries?q=02", actor={"id": "root"}, ) + flags_response = await ds.client.get( + "/data/-/queries", + actor={"id": "root"}, + ) json_response = await ds.client.get( "/data/-/queries.json?q=02", actor={"id": "root"}, ) filtered_response = await ds.client.get( - "/data/-/queries.json?is_published=0", + "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) @@ -453,7 +608,22 @@ async def test_query_list_search_filter_and_html(): assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text assert "Mode" in html_response.text - assert 'type="radio" name="is_published" value="1"' in html_response.text + assert 'type="radio" name="is_private" value="1"' in html_response.text + assert "Only the owning actor can view this query." not in html_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + not in html_response.text + ) + assert flags_response.status_code == 200 + assert 'Owner' in flags_response.text + assert 'Flags' in flags_response.text + assert 'Mode' not in flags_response.text + assert 'class="query-list-owner">root' in flags_response.text + assert 'class="query-list-pill">Read-only' in flags_response.text + assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text + assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert "Only the owning actor can view this query." in flags_response.text + assert "Execution skips the usual SQL and write permission checks" in flags_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" @@ -491,7 +661,6 @@ async def test_global_query_list_api_and_html(): "alpha_first", "select 1", title="Alpha first", - is_published=True, source="user", owner_id="root", ) @@ -500,7 +669,6 @@ async def test_global_query_list_api_and_html(): "alpha_second", "select 2", title="Alpha second", - is_published=True, source="user", owner_id="root", ) @@ -509,7 +677,6 @@ async def test_global_query_list_api_and_html(): "beta_first", "select 3", title="Beta first", - is_published=True, source="user", owner_id="root", ) @@ -548,7 +715,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_publish_requires_publish_query(): +async def test_query_insert_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -564,17 +731,17 @@ async def test_query_insert_api_publish_requires_publish_query(): } }, ) - ds.add_memory_database("query_publish_api", name="data") + ds.add_memory_database("query_trusted_api", name="data") await ds.invoke_startup() response = await ds.client.post( "/data/-/queries/insert", actor={"id": "writer"}, - json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, + json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) - assert response.status_code == 403 - assert response.json()["errors"] == ["Permission denied: need publish-query"] + assert response.status_code == 400 + assert response.json()["errors"] == ["Invalid keys: is_trusted"] @pytest.mark.asyncio @@ -599,24 +766,10 @@ async def test_query_insert_api_creates_writable_query(): assert response.status_code == 201 query = response.json()["query"] assert query["is_write"] is True - assert query["is_published"] is False + assert query["is_private"] is True + assert query["is_trusted"] is False assert query["parameters"] == ["name"] - bad_response = await ds.client.post( - "/data/-/queries/insert", - actor={"id": "root"}, - json={ - "query": { - "name": "published_insert", - "sql": "insert into dogs (name) values (:name)", - "is_published": True, - } - }, - ) - - assert bad_response.status_code == 400 - assert bad_response.json()["errors"] == ["Writable queries cannot be published"] - @pytest.mark.asyncio async def test_query_update_and_delete_api(): @@ -1103,6 +1256,10 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): config={ "databases": { "data": { + "permissions": { + "view-database": {"id": ["alice", "bob"]}, + "execute-write-sql": {"id": ["alice", "bob"]}, + }, "tables": { "dogs": { "permissions": { From 1cd162e9da48b924c289ec9343e9d801b51a89f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:07:30 -0700 Subject: [PATCH 1760/1866] Removed some no-longer-necessary code, simplified view-query is back in the default allow actions now. We have other mechanisms that work for controlling visibility, and the fact that queries default to running with the permissions of the actor makes this safe. --- datasette/default_permissions/defaults.py | 55 +++-------------------- tests/test_permissions.py | 9 +++- tests/test_queries.py | 39 ++++++++++++++++ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index dfd8d3e9..ed0a6d66 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -21,37 +21,12 @@ DEFAULT_ALLOW_ACTIONS = frozenset( "view-database", "view-database-download", "view-table", + "view-query", "execute-sql", } ) -def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: - selects = [] - params = {} - for index, (database_name, db_config) in enumerate( - ((datasette.config or {}).get("databases") or {}).items() - ): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if isinstance(query_config, dict) and query_config.get("is_private"): - continue - parent_param = f"query_config_parent_{index}_{len(selects)}" - child_param = f"query_config_child_{index}_{len(selects)}" - selects.append( - f""" - SELECT :{parent_param} AS parent, :{child_param} AS child - WHERE NOT EXISTS ( - SELECT 1 FROM queries - WHERE database_name = :{parent_param} - AND name = :{child_param} - ) - """ - ) - params[parent_param] = database_name - params[child_param] = query_name - return selects, params - - @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -121,16 +96,6 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] - if not datasette.default_deny: - rule_sqls.append( - """ - SELECT database_name AS parent, name AS child, 1 AS allow, - 'non-private query' AS reason - FROM queries - WHERE is_private = 0 - """ - ) - if actor_id is not None: rule_sqls.append( """ @@ -141,23 +106,13 @@ async def default_query_permissions_sql( """ ) - config_restriction_selects, config_restriction_params = ( - _configured_query_restriction_selects(datasette) - ) - - restriction_sqls = [ - """ + return PermissionSQL( + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql=""" SELECT database_name AS parent, name AS child FROM queries WHERE is_private = 0 OR owner_id = :query_owner_id - """ - ] - restriction_sqls.extend(config_restriction_selects) - params.update(config_restriction_params) - - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql="\nUNION ALL\n".join(restriction_sqls), + """, params=params, ) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 22f294bb..4f342d8f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,16 +937,20 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config + await perms_ds.apply_queries_config() try: # Convert old-style resource to Resource object - from datasette.resources import DatabaseResource, TableResource + from datasette.resources import DatabaseResource, QueryResource, TableResource resource_obj = None if resource: if isinstance(resource, str): resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: - resource_obj = TableResource(database=resource[0], table=resource[1]) + if action == "view-query": + resource_obj = QueryResource(database=resource[0], query=resource[1]) + else: + resource_obj = TableResource(database=resource[0], table=resource[1]) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor @@ -956,6 +960,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config + await perms_ds.apply_queries_config() @pytest.mark.asyncio diff --git a/tests/test_queries.py b/tests/test_queries.py index c97b5733..dde57dea 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -248,6 +248,45 @@ async def test_default_deny_blocks_view_query_even_for_trusted_query(): ) +@pytest.mark.asyncio +async def test_view_query_default_allow_still_respects_private_restriction(): + ds = Datasette(memory=True) + ds.add_memory_database("default_view_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "shared_report"), + actor=None, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + @pytest.mark.asyncio async def test_private_query_restriction_blocks_broad_view_query_permission(): ds = Datasette( From 1ac4265ffd295ea62008b13b3e37af96f5450be4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:12:59 -0700 Subject: [PATCH 1761/1866] Require permissions for untrusted stored query execution, refs #2735 --- datasette/views/database.py | 7 +++---- docs/authentication.rst | 2 +- queries-plan.md | 8 +++----- tests/test_queries.py | 12 ++++++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 91e9c350..bd939d87 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1430,10 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write"): - await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor - ) + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) # If database is immutable, return an error if not db.is_mutable: diff --git a/docs/authentication.rst b/docs/authentication.rst index 6e835c8d..453aaa19 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view (and execute) a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. +Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/queries-plan.md b/queries-plan.md index f4b8049c..da6b7c92 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -25,7 +25,7 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. - `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. - `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages execute if the actor has `view-query` for `QueryResource(database, query)`. +- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. - Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. @@ -145,9 +145,7 @@ Default execution rule for user-created writable queries: Implementation: -- Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. -- Replace it with query-aware default `view-query` permission SQL. -- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. - Emit default `view-query` allows for the owning actor. - Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. - Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. @@ -424,7 +422,7 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - The old `canned_queries()` hook is no longer called by core. - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. -- `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. +- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. - Private query is only visible to its owner, even when a broader `view-query` rule applies. - Non-trusted read-only query requires `execute-sql` to execute. - Trusted read-only query can be executed without `execute-sql` after `view-query` passes. diff --git a/tests/test_queries.py b/tests/test_queries.py index dde57dea..997f8b39 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,8 +395,16 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) - assert denied.status_code == 403 + denied_get = await ds.client.get( + "/data/shared_report.json", actor={"id": "viewer"} + ) + denied_post = await ds.client.post( + "/data/shared_report", + actor={"id": "viewer"}, + data={}, + ) + assert denied_get.status_code == 403 + assert denied_post.status_code == 403 ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) From 866852eff603c219b8bf7d13f2a69b5ff032fa67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:46:18 -0700 Subject: [PATCH 1762/1866] Clarifying comments --- datasette/default_permissions/defaults.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index ed0a6d66..32ad4ef1 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -80,6 +80,7 @@ async def default_query_permissions_sql( if action in {"update-query", "delete-query"}: if actor_id is None: return None + # Query owner can update/delete query return PermissionSQL( sql=""" SELECT database_name AS parent, name AS child, 1 AS allow, @@ -97,15 +98,15 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - rule_sqls.append( - """ + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id - """ - ) + """) + # restriction_sql enforces private queries ONLY visible to owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" From 71c76e38534378cbce8576771238a788feccf3ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:08:19 -0700 Subject: [PATCH 1763/1866] Better faceting on /-/queries Ref https://github.com/simonw/datasette/pull/2741#issuecomment-4548321815 --- datasette/app.py | 69 +++++++++++++++++ datasette/templates/query_list.html | 94 +++++++++++++---------- datasette/views/database.py | 99 +++++++++++++++++++++++- tests/test_permissions.py | 8 +- tests/test_queries.py | 115 +++++++++++++++++++++++++--- 5 files changed, 330 insertions(+), 55 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3329ee7e..1acdfcd8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1298,6 +1298,75 @@ class Datasette: ) return self._query_row_to_dict(rows.first()) + async def count_queries( + self, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + ): + allowed_sql, allowed_params = await self.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await self.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + async def list_queries( self, database=None, diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index 25259b3d..fa4859b1 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -9,7 +9,7 @@ max-width: 64rem; } .query-list-filters { - margin: 0.5rem 0 1rem; + margin: 0.5rem 0 0.75rem; } .query-list-search { align-items: center; @@ -32,43 +32,63 @@ line-height: 1.1; padding: 0.35rem 0.65rem; } -.query-list-filter-groups { +.query-list-facets { align-items: flex-start; display: flex; flex-wrap: wrap; - gap: 0.8rem 1.4rem; + gap: 1rem 1.6rem; + margin: 0 0 1rem; } -.query-list-filter-group { - border: 0; +.query-list-facet { + margin: 0; +} +.query-list-facet h2 { + font-size: 0.9rem; + line-height: 1.2; + margin: 0 0 0.35rem; +} +.query-list-facet ul { display: flex; flex-wrap: wrap; gap: 0.35rem; margin: 0; - min-width: 0; padding: 0; + list-style: none; } -.query-list-filter-group legend { - font-weight: 700; - margin: 0 0.45rem 0 0; - padding: 0; -} -.query-list-filter-group label { +.query-list-facet-link, +.query-list-facet-link:link, +.query-list-facet-link:visited, +.query-list-facet-link:hover, +.query-list-facet-link:focus, +.query-list-facet-link:active { align-items: center; border: 1px solid #c8d1dc; border-radius: 0.25rem; - cursor: pointer; + color: #39445a; display: inline-flex; font-size: 0.82rem; - gap: 0.3rem; + gap: 0.4rem; line-height: 1.1; padding: 0.35rem 0.55rem; + text-decoration: none; } -.query-list-filter-group input { - margin: 0; +.query-list-facet-link:hover { + border-color: #7ca5c8; + color: #1f5d85; } -.query-list-filter-group input:checked + span { +.query-list-facet-link-active { + background-color: #edf6fb; + border-color: #6d9fc0; font-weight: 700; } +.query-list-facet-disabled { + color: #7b8794; + cursor: default; +} +.query-list-facet-count { + color: #4f5b6d; + font-variant-numeric: tabular-nums; +} .query-list-results { border-collapse: collapse; font-size: 0.9rem; @@ -169,15 +189,6 @@ .query-list-search input[type=search] { max-width: none; } - .query-list-filter-group { - display: block; - } - .query-list-filter-group legend { - margin-bottom: 0.3rem; - } - .query-list-filter-group label { - margin: 0 0.25rem 0.35rem 0; - } } {% endblock %} @@ -198,24 +209,27 @@

    + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %}

    -
    -
    - Mode - - - -
    -
    - Visibility - - - -
    -
    + + {% if queries %}
    diff --git a/datasette/views/database.py b/datasette/views/database.py index bd939d87..2e77d36b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1121,6 +1121,21 @@ class QueryParametersView(BaseView): return _block_framing(Response.json({"ok": True, "parameters": parameters})) +def _query_list_url(path, query_string, *, set_args=None, remove_args=None): + set_args = set_args or {} + remove_args = set(remove_args or ()) + skip = set(set_args) | remove_args | {"_next"} + pairs = [ + (key, value) + for key, value in parse_qsl(query_string, keep_blank_values=True) + if key not in skip + ] + for key, value in set_args.items(): + if value not in (None, ""): + pairs.append((key, value)) + return path + (("?" + urlencode(pairs)) if pairs else "") + + class QueryListView(BaseView): name = "query-list" @@ -1139,9 +1154,7 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_private = _as_optional_bool( - request.args.get("is_private"), "is_private" - ) + is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1173,6 +1186,80 @@ class QueryListView(BaseView): urlencode(pairs), ) + current_filters = { + "actor": request.actor, + "q": request.args.get("q") or None, + "is_write": is_write, + "is_private": is_private, + "source": request.args.get("source") or None, + "owner_id": request.args.get("owner_id") or None, + } + + async def facet_count(field, value): + if current_filters[field] is not None and current_filters[field] != value: + return 0 + filters = dict(current_filters) + filters[field] = value + return await self.ds.count_queries(database, **filters) + + def facet_href(field, value): + if current_filters[field] == value: + return _query_list_url( + query_list_path, + request.query_string, + remove_args=[field], + ) + if current_filters[field] is not None: + return None + return _query_list_url( + query_list_path, + request.query_string, + set_args={field: str(int(value))}, + ) + + async def facet_item(label, field, value): + count = await facet_count(field, value) + active = current_filters[field] == value + if not active and not count: + return None + return { + "label": label, + "count": count, + "href": facet_href(field, value) if active or count else None, + "active": active, + } + + async def facet_items(items): + return [ + item + for item in [ + await facet_item(label, field, value) + for label, field, value in items + ] + if item is not None + ] + + facets = [ + { + "title": "Mode", + "items": await facet_items( + [ + ("Read-only", "is_write", False), + ("Writable", "is_write", True), + ] + ), + }, + { + "title": "Visibility", + "items": await facet_items( + [ + ("Not private", "is_private", False), + ("Private", "is_private", True), + ] + ), + }, + ] + data = { "ok": True, "database": database, @@ -1188,6 +1275,7 @@ class QueryListView(BaseView): "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, + "facets": facets, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1715,6 +1803,9 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) + if canned_query: + metadata = dict(canned_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): @@ -1865,7 +1956,7 @@ class QueryView(View): ) ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=canned_query or metadata, + metadata=metadata, alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4f342d8f..eb6cee9f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -948,9 +948,13 @@ async def test_permissions_in_config( resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: if action == "view-query": - resource_obj = QueryResource(database=resource[0], query=resource[1]) + resource_obj = QueryResource( + database=resource[0], query=resource[1] + ) else: - resource_obj = TableResource(database=resource[0], table=resource[1]) + resource_obj = TableResource( + database=resource[0], table=resource[1] + ) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor diff --git a/tests/test_queries.py b/tests/test_queries.py index 997f8b39..36f7107a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,9 +395,7 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied_get = await ds.client.get( - "/data/shared_report.json", actor={"id": "viewer"} - ) + denied_get = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) denied_post = await ds.client.post( "/data/shared_report", actor={"id": "viewer"}, @@ -608,6 +606,27 @@ async def test_query_list_and_definition_api(): assert definition_response.json()["query"]["title"] == "Demo query 01" +@pytest.mark.asyncio +async def test_query_page_does_not_show_internal_source(): + ds = Datasette(memory=True) + ds.add_memory_database("query_page_source", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "stored_report", + "select 1 as one", + title="Stored report", + source="user", + owner_id="root", + ) + + response = await ds.client.get("/data/stored_report", actor={"id": "root"}) + + assert response.status_code == 200 + assert "Stored report" in response.text + assert "Data source:" not in response.text + + @pytest.mark.asyncio async def test_query_list_search_filter_and_html(): ds = Datasette(memory=True) @@ -632,6 +651,15 @@ async def test_query_list_search_filter_and_html(): is_trusted=True, source="config", ) + await ds.add_query( + "data", + "writable_query", + "insert into dogs (name) values (:name)", + title="Writable query", + is_write=True, + source="user", + owner_id="root", + ) html_response = await ds.client.get( "/data/-/queries?q=02", @@ -649,13 +677,21 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) + filtered_write_response = await ds.client.get( + "/data/-/queries?is_write=1", + actor={"id": "root"}, + ) + filtered_private_response = await ds.client.get( + "/data/-/queries?is_private=1", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text - assert "Mode" in html_response.text - assert 'type="radio" name="is_private" value="1"' in html_response.text + assert 'class="query-list-facets"' in html_response.text + assert 'type="radio"' not in html_response.text assert "Only the owning actor can view this query." not in html_response.text assert ( "Execution skips the usual SQL and write permission checks" @@ -667,14 +703,75 @@ async def test_query_list_search_filter_and_html(): assert '' not in flags_response.text assert 'class="query-list-owner">root' in flags_response.text assert 'class="query-list-pill">Read-only' in flags_response.text - assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text - assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert ( + 'class="query-list-pill query-list-pill-write">Writable' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-private">Private' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-trusted">Trusted' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=0">Read-only5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1">Writable1' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=0">Not private5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=1">Private1' + in flags_response.text + ) assert "Only the owning actor can view this query." in flags_response.text - assert "Execution skips the usual SQL and write permission checks" in flags_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + in flags_response.text + ) assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] + assert "Writable query" in filtered_write_response.text + assert "Demo query 01" not in filtered_write_response.text + assert ( + 'query-list-facet-link query-list-facet-link-active" href="/data/-/queries"' + in filtered_write_response.text + ) + assert ( + 'Read-only0' + not in filtered_write_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1&is_private=0">Not private1' + in filtered_write_response.text + ) + assert ( + 'Private0' + not in filtered_write_response.text + ) + assert "Private query" in filtered_private_response.text + assert "Demo query 01" not in filtered_private_response.text + assert ( + 'href="/data/-/queries?is_private=1&is_write=0">Read-only1' + in filtered_private_response.text + ) + assert ( + 'Writable0' + not in filtered_private_response.text + ) + assert ( + 'Not private0' + not in filtered_private_response.text + ) @pytest.mark.asyncio @@ -1313,7 +1410,7 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): "insert-row": {"id": "alice"}, } } - } + }, } } }, From 0fcaa5792ba73143661515af0088d7e5d968e96c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:12:07 -0700 Subject: [PATCH 1764/1866] Style query operations on create query Made it consistent with the SQL write page. --- .../_execute_write_analysis_styles.html | 37 +++++++++++++++++++ datasette/templates/execute_write.html | 36 +----------------- datasette/templates/query_create.html | 19 +++++----- tests/test_queries.py | 6 ++- 4 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_styles.html diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..f20e67b2 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,37 @@ + diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 46f58c3b..414d4af7 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -40,42 +40,8 @@ border-radius: 0.25rem; min-width: 13rem; } -.execute-write-analysis { - border-collapse: collapse; - font-size: 0.9rem; - margin: 0.25rem 0 1rem; - min-width: 44rem; -} -.execute-write-analysis th, -.execute-write-analysis td { - border-bottom: 1px solid #d7dde5; - padding: 0.45rem 0.7rem; - text-align: left; - vertical-align: top; -} -.execute-write-analysis th { - background-color: #edf6fb; - border-top: 1px solid #d7dde5; - color: #39445a; - font-weight: 700; -} -.execute-write-analysis tbody tr:nth-child(even) { - background-color: rgba(39, 104, 144, 0.05); -} -.execute-write-analysis code { - background: transparent; - font-size: 0.9em; - white-space: nowrap; -} -.execute-write-analysis-allowed { - color: #267a3e; - font-weight: 700; -} -.execute-write-analysis-denied { - color: #b00020; - font-weight: 700; -} +{% include "_execute_write_analysis_styles.html" %} {% include "_sql_parameter_styles.html" %} {% endblock %} diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 686d971e..2d8a9122 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -5,6 +5,7 @@ {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} +{% include "_execute_write_analysis_styles.html" %} {% endblock %} {% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} @@ -32,30 +33,28 @@

    Execute write SQL

    {% endif %} -

    Analysis

    +

    Query operations

    {% if analysis_error %}

    {{ analysis_error }}

    {% elif analysis_rows %} -
    Mode
    +
    - + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} diff --git a/tests/test_queries.py b/tests/test_queries.py index 36f7107a..c27c23da 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -998,7 +998,11 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert "Create query" in create_response.text assert "Read-only" in create_response.text assert "Writable" in create_response.text - assert "required permission" in create_response.text + assert "

    Query operations

    " in create_response.text + assert '
    Operation Database Tablerequired permissionRequired permission AllowedSource
    {{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
    ' in create_response.text + assert '' in create_response.text + assert '' not in create_response.text + assert "" in create_response.text assert query_response.status_code == 200 assert "Save query" in query_response.text assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text From 70b23ff4a55528083512fab96aa50725f415cbe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:47:24 -0700 Subject: [PATCH 1765/1866] Tweaked save query link --- datasette/templates/query.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f74d21f1..1900bd31 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -66,7 +66,7 @@ {% if not hide_sql %}{% endif %} {{ show_hide_hidden }} - {% if save_query_url %}Save query{% endif %} + {% if save_query_url %}Save this query{% endif %} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

    From eb7c25c57cf914629c08eaa477d0709b0f41efeb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:48:40 -0700 Subject: [PATCH 1766/1866] Major redesign of create saved query UI https://github.com/simonw/datasette/pull/2741#issuecomment-4548707129 --- datasette/app.py | 6 +- datasette/static/app.css | 4 + .../_execute_write_analysis_scripts.html | 111 +++++++ .../_execute_write_analysis_styles.html | 4 + .../templates/_sql_parameter_scripts.html | 17 +- datasette/templates/execute_write.html | 88 +----- datasette/templates/query_create.html | 296 +++++++++++++++--- datasette/views/database.py | 181 ++++++++--- tests/test_queries.py | 170 +++++++++- 9 files changed, 705 insertions(+), 172 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_scripts.html diff --git a/datasette/app.py b/datasette/app.py index 1acdfcd8..8936b099 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,7 +50,7 @@ from .views.database import ( ExecuteWriteView, TableCreateView, QueryView, - QueryCreateView, + QueryCreateAnalyzeView, QueryDeleteView, QueryDefinitionView, GlobalQueryListView, @@ -2820,8 +2820,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", ) add_route( - QueryCreateView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/create$", + QueryCreateAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( QueryInsertView.as_view(self), diff --git a/datasette/static/app.css b/datasette/static/app.css index c21d0dc4..4f4db133 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1414,6 +1414,10 @@ svg.dropdown-menu-icon { position: relative; top: 1px; } +.save-query { + display: inline-block; + margin-left: 0.45em; +} .blob-download { display: block; diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html index f20e67b2..165cfe9f 100644 --- a/datasette/templates/_execute_write_analysis_styles.html +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -34,4 +34,8 @@ color: #b00020; font-weight: 700; } +.execute-write-analysis-na { + color: #687386; + font-style: italic; +} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 68e46069..159a141c 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -215,9 +215,10 @@ window.datasetteSqlParameters = (() => { if (!form) { return null; } + const shouldRenderParameters = options.renderParameters !== false; const section = options.section || form.querySelector("[data-sql-parameters-section]"); - if (!section) { + if (shouldRenderParameters && !section) { return null; } const manager = { @@ -225,12 +226,16 @@ window.datasetteSqlParameters = (() => { section, allowExpand: options.allowExpand === undefined - ? section.dataset.allowExpand === "1" + ? section + ? section.dataset.allowExpand === "1" + : false : options.allowExpand, parameterState: new Map(), }; - bindParameterControls(manager); - syncParameterState(manager); + if (section) { + bindParameterControls(manager); + syncParameterState(manager); + } const url = options.url || form.dataset.parametersUrl; let refreshTimer = null; @@ -254,7 +259,9 @@ window.datasetteSqlParameters = (() => { if (!response.ok) { throw new Error((data.errors || [response.statusText]).join("; ")); } - renderParameters(manager, data.parameters || []); + if (shouldRenderParameters) { + renderParameters(manager, data.parameters || []); + } if (options.onData) { options.onData(data, manager); } diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 414d4af7..7a627a7a 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -131,6 +131,7 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} {% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2e77d36b..aafcf40b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -551,6 +551,17 @@ def _wants_json(request, is_json, data): ) +def _query_create_form_error_message(message): + return { + "Query name is required": "URL is required", + "Invalid query name": "Invalid URL", + "Query name conflicts with a table or view": ( + "URL conflicts with an existing table or view" + ), + "Query already exists": "A query already exists at that URL", + }.get(message, message) + + async def _json_or_form_payload(request): content_type = request.headers.get("content-type", "") if content_type.startswith("application/json"): @@ -731,6 +742,54 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): } +async def _query_create_analysis_data(datasette, db, sql, actor): + has_sql = bool(sql and sql.strip()) + parameter_names = [] + analysis_rows = [] + analysis_error = None + if has_sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "has_sql": has_sql, + "analysis_is_write": bool( + analysis_rows and any(row["required_permission"] for row in analysis_rows) + ), + "save_disabled": bool( + (not has_sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + +async def _query_create_form_context( + datasette, request, db, *, sql="", name="", title="", description="", is_private=True +): + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "sql": sql, + "name": name, + "title": title, + "description": description, + "is_private": is_private, + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1307,6 +1366,35 @@ class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False + async def _render_form( + self, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, + status=200, + ): + response = await self.render( + ["query_create.html"], + request, + await _query_create_form_context( + self.ds, + request, + db, + sql=sql, + name=name, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + async def get(self, request): db = await self.ds.resolve_database(request) await self.ds.ensure_permission( @@ -1320,46 +1408,61 @@ class QueryCreateView(BaseView): actor=request.actor, ) - sql = request.args.get("sql") or "" - analysis_error = None - analysis_rows = [] - parameter_names = [] - if sql: - try: - parameter_names = _derived_query_parameters(sql) - params = {parameter: "" for parameter in parameter_names} - analysis = await db.analyze_sql(sql, params) - analysis_rows = await _analysis_rows_with_permissions( - self.ds, analysis, request.actor - ) - except (QueryValidationError, sqlite3.DatabaseError) as ex: - analysis_error = getattr(ex, "message", str(ex)) + return await self._render_form(request, db, sql=request.args.get("sql") or "") - return await self.render( - ["query_create.html"], - request, - { - "database": db.name, - "database_color": db.color, - "sql": sql, - "parameter_names": parameter_names, - "analysis_error": analysis_error, - "analysis_rows": analysis_rows, - "analysis_is_write": bool( - analysis_rows - and any(row["required_permission"] for row in analysis_rows) - ), - "save_disabled": bool( - analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), - }, + +class QueryCreateAnalyzeView(BaseView): + name = "query-create-analyze" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + if not await self.ds.allowed( + action="insert-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need insert-query"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = request.args.get("sql") or "" + return _block_framing( + Response.json( + await _query_create_analysis_data(self.ds, db, sql, request.actor) + ) ) -class QueryInsertView(BaseView): +class QueryInsertView(QueryCreateView): name = "query-insert" + async def _error_response(self, request, db, query_data, message, status): + message = _query_create_form_error_message(message) + self.ds.add_message(request, message, self.ds.ERROR) + return await self._render_form( + request, + db, + sql=query_data.get("sql") or "", + name=query_data.get("name") or "", + title=query_data.get("title") or "", + description=query_data.get("description") or "", + is_private=_as_bool(query_data.get("is_private", True)), + status=status, + ) + async def post(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( @@ -1375,6 +1478,8 @@ class QueryInsertView(BaseView): ): return _error(["Permission denied: need insert-query"], 403) + is_json = False + query_data = {} try: data, is_json = await _json_or_form_payload(request) if not isinstance(data, dict): @@ -1384,6 +1489,10 @@ class QueryInsertView(BaseView): raise QueryValidationError("JSON must contain a query dictionary") prepared = await _prepare_query_create(self.ds, request, db, query_data) except QueryValidationError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response( + request, db, query_data, ex.message, ex.status + ) return _error([ex.message], ex.status) prepared.pop("analysis") @@ -1391,6 +1500,8 @@ class QueryInsertView(BaseView): try: await self.ds.add_query(db.name, name, replace=False, **prepared) except sqlite3.IntegrityError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response(request, db, query_data, str(ex), 400) return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) @@ -1896,7 +2007,7 @@ class QueryView(View): ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/-/create?" + + "/-/queries/insert?" + urlencode({"sql": sql}) ) diff --git a/tests/test_queries.py b/tests/test_queries.py index c27c23da..32cdfae3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -986,6 +986,14 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + blank_create_response = await ds.client.get( + "/data/-/queries/insert", + actor={"id": "root"}, + ) + old_create_response = await ds.client.get( "/data/-/queries/-/create?sql=select+*+from+dogs", actor={"id": "root"}, ) @@ -996,16 +1004,171 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert create_response.status_code == 200 assert "Create query" in create_response.text - assert "Read-only" in create_response.text assert "Writable" in create_response.text + assert 'type="radio"' not in create_response.text + assert 'name="parameters"' not in create_response.text + assert 'id="query-parameters"' not in create_response.text + assert 'class="query-create-field"' in create_response.text + assert '' not in create_response.text + assert '' in create_response.text + assert '' in create_response.text + assert '/data/' in create_response.text + assert ( + '' + in create_response.text + ) + assert 'function slugify(value)' in create_response.text + assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text + assert "setupSqlParameterRefresh" in create_response.text + assert "renderParameters: false" in create_response.text + assert "datasetteSqlAnalysis.renderAnalysis" in create_response.text + assert "data-query-create-submit" in create_response.text + assert "data-query-create-writable" in create_response.text + assert ( + "Queries marked private can only be seen by you, their creator." + in create_response.text + ) assert "

    Query operations

    " in create_response.text assert '
    Required permissionSourceread
    ' in create_response.text assert '' in create_response.text assert '' not in create_response.text assert "" in create_response.text + assert ( + create_response.text.count( + '' + ) + == 2 + ) + assert create_response.text.index('value="Save query"') < create_response.text.index( + "

    Query operations

    " + ) + assert blank_create_response.status_code == 200 + assert ( + '
    Required permissionSourcereadn/a
    ' in response.text assert '' in response.text assert "" in response.text From 5dca2dc9beea96c52e6a9c806df66c9a1f2f7874 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:54:47 -0700 Subject: [PATCH 1767/1866] Show query count on database page --- datasette/templates/database.html | 2 +- datasette/views/database.py | 18 +++++++++++++++++- tests/test_queries.py | 11 ++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 62f9c620..371f6a22 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -59,7 +59,7 @@ {% endfor %} {% if queries_more %} -

    View all queries

    +

    View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

    {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index feb38619..d40d69d1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -102,6 +102,11 @@ class DatabaseView(View): ) canned_queries = queries_page["queries"] queries_more = queries_page["has_more"] + queries_count = ( + await datasette.count_queries(database, actor=request.actor) + if queries_more + else len(canned_queries) + ) async def database_actions(): links = [] @@ -134,6 +139,7 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "queries_more": queries_more, + "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -168,6 +174,7 @@ class DatabaseView(View): views=sql_views, queries=canned_queries, queries_more=queries_more, + queries_count=queries_count, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -219,6 +226,7 @@ class DatabaseContext(Context): queries_more: bool = field( metadata={"help": "Boolean indicating if more saved queries are available"} ) + queries_count: int = field(metadata={"help": "Count of visible saved queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -775,7 +783,15 @@ async def _query_create_analysis_data(datasette, db, sql, actor): async def _query_create_form_context( - datasette, request, db, *, sql="", name="", title="", description="", is_private=True + datasette, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, ): analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) return { diff --git a/tests/test_queries.py b/tests/test_queries.py index 32cdfae3..09b41645 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -458,9 +458,10 @@ async def test_database_page_query_preview_is_limited(): assert html_response.status_code == 200 assert "Demo query 05" in html_response.text assert "Demo query 06" not in html_response.text - assert 'href="/data/-/queries"' in html_response.text + assert 'View 25 queries' in html_response.text assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True + assert json_response.json()["queries_count"] == 25 @pytest.mark.asyncio @@ -1017,7 +1018,7 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): '' in create_response.text ) - assert 'function slugify(value)' in create_response.text + assert "function slugify(value)" in create_response.text assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text assert "setupSqlParameterRefresh" in create_response.text assert "renderParameters: false" in create_response.text @@ -1039,9 +1040,9 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) == 2 ) - assert create_response.text.index('value="Save query"') < create_response.text.index( - "

    Query operations

    " - ) + assert create_response.text.index( + 'value="Save query"' + ) < create_response.text.index("

    Query operations

    ") assert blank_create_response.status_code == 200 assert ( '
    Required permissioninsert
    ' in create_response.text assert '' in create_response.text @@ -1053,6 +1067,12 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): "

    Analysis will show each affected table and required permission.

    " not in blank_create_response.text ) + assert "Enter SQL to analyze this query." in blank_create_response.text + assert write_create_response.status_code == 200 + assert ( + 'This query updates data in the database.' + in write_create_response.text + ) assert query_response.status_code == 200 assert "Save this query" in query_response.text assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text From 024b9117725bbed17396a5a4b3f48663c23337f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:09:53 -0700 Subject: [PATCH 1769/1866] Clarifying comment https://github.com/simonw/datasette/pull/2741/changes#r3306856046 --- datasette/default_permissions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index a9f2d8bd..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -26,6 +26,7 @@ from .restrictions import ( from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, default_query_permissions_sql as default_query_permissions_sql, From ac6ee097dd06050188d44c6d4b17a98a12c7b481 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:10:48 -0700 Subject: [PATCH 1770/1866] Disallow update/delete of private queries If a user does not own a private query they cannot update or delete it either, even if they have global update-query. https://github.com/simonw/datasette/pull/2741/changes#r3306417463 --- datasette/default_permissions/defaults.py | 33 ++++----- tests/test_queries.py | 81 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 32ad4ef1..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -77,36 +77,31 @@ async def default_query_permissions_sql( ) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None - if action in {"update-query", "delete-query"}: - if actor_id is None: - return None - # Query owner can update/delete query - return PermissionSQL( - sql=""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """, - params={"query_owner_id": actor_id}, - ) - - if action != "view-query": + if action not in {"view-query", "update-query", "delete-query"}: return None params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - # Query owner can view-query - rule_sqls.append(""" + if action in {"update-query", "delete-query"}: + # Query owner can update/delete query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """) + else: + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id """) - # restriction_sql enforces private queries ONLY visible to owner + # restriction_sql enforces private queries ONLY visible/mutable by owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" diff --git a/tests/test_queries.py b/tests/test_queries.py index f888dda0..26a0748c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1581,6 +1581,87 @@ async def test_query_owner_gets_update_delete_and_writable_view_defaults(): ) +@pytest.mark.asyncio +async def test_private_query_restricts_broad_update_delete_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "update-query": {"id": "bob"}, + "delete-query": {"id": "bob"}, + }, + }, + }, + }, + ) + ds.add_memory_database("query_broad_update_delete", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "alice_private", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "alice_public", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + for action in ("update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "bob"}, + ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), + actor={"id": "bob"}, + ) + + private_update_response = await ds.client.post( + "/data/alice_private/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Nope"}}, + ) + private_delete_response = await ds.client.post( + "/data/alice_private/-/delete", + actor={"id": "bob"}, + json={}, + ) + public_update_response = await ds.client.post( + "/data/alice_public/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Bob can edit public queries"}}, + ) + public_delete_response = await ds.client.post( + "/data/alice_public/-/delete", + actor={"id": "bob"}, + json={}, + ) + + assert private_update_response.status_code == 403 + assert private_delete_response.status_code == 403 + assert public_update_response.status_code == 200 + assert public_delete_response.status_code == 200 + assert await ds.get_query("data", "alice_private") is not None + assert await ds.get_query("data", "alice_public") is None + + @pytest.mark.asyncio async def test_user_writable_query_execution_rechecks_table_permissions(): ds = Datasette( From 180a6a86fd77ac43f6cf3bfb7d7f9150003da419 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:16:10 -0700 Subject: [PATCH 1771/1866] Remove queries-plan.md We do not need this any more. It can live forever in Git history. --- queries-plan.md | 446 ------------------------------------------------ 1 file changed, 446 deletions(-) delete mode 100644 queries-plan.md diff --git a/queries-plan.md b/queries-plan.md deleted file mode 100644 index da6b7c92..00000000 --- a/queries-plan.md +++ /dev/null @@ -1,446 +0,0 @@ -# Queries in the internal database - -Plan for . - -## Goal - -Move named query definitions into Datasette's internal database, so hundreds or thousands of queries can be listed, searched, permission-filtered, managed, and executed efficiently. - -Terminology change: these are now "queries", not "canned queries". Legacy code and documentation can mention the old name only when describing compatibility or migration. - -## Decisions so far - -- Internal table name: `queries`. -- Query definitions should use real columns, not a JSON blob for all options. -- Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No separate index is needed for the privacy/trust flags yet. -- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. -- Add `update-query` and `delete-query`, so administrators can manage queries created by other users. -- Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. -- Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. - -## Current shape - -- Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. -- `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. -- `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. -- Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. - -The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. - -## Proposed internal schema - -Start with one `queries` table. - -```sql -CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) -); - -CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); -``` - -Column notes: - -- `database_name`, `name`, and `sql` are the routing and execution core. -- Display fields become columns: `title`, `description`, and `description_html`. -- Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. -- `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. -- Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. -- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. -- `source` distinguishes `user`, `config`, and `plugin` rows. -- `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. - -No separate index is needed on `(database_name, name)` because the primary key already creates one. - -`QueryResource.resources_sql()` can become: - -```sql -SELECT q.database_name AS parent, q.name AS child -FROM queries q -JOIN catalog_databases cd ON cd.database_name = q.database_name -``` - -The join keeps persisted queries for detached databases from appearing as live resources. - -## Config and plugin migration - -`datasette.yaml` can continue to support `databases: {db}: queries:` blocks, but core should import them directly into the internal `queries` tables at startup: - -1. Ensure the internal schema exists. -2. Delete previous `source='config'` rows. -3. Read configured query blocks for each live database. -4. Normalize string definitions to `{"sql": ...}`. -5. Insert rows into `queries`, storing explicit `params` as JSON in `parameters`. - -Plugins should move to: - -```python -await datasette.add_query(...) -await datasette.remove_query(...) -``` - -Remove the old `canned_queries()` hookspec and all core calls to it. If compatibility is needed, build `datasette-old-canned-queries` later as a plugin that restores the hook and imports old hook results using `datasette.add_query()`. - -## Permission model - -Add core actions: - -- `insert-query`, database-level, for creating queries in a database. -- `update-query`, query-level, for modifying existing query definitions. -- `delete-query`, query-level, for deleting existing query definitions. - -User-created query creation requires: - -- `execute-sql` on `DatabaseResource(database)` -- `insert-query` on `DatabaseResource(database)` -- If analysis shows the query is writable, the table-level write permissions described in the writable query section. - -Updating an existing query requires: - -- `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. -- If the SQL changes, also require `execute-sql` on the database. -- If the changed SQL is writable, also require the table-level write permissions described in the writable query section. - -Deleting an existing query requires: - -- `delete-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - -Default owner permissions: - -- For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. - -## Executing queries - -Default execution rule for read-only queries: - -- If `is_trusted=0`, the actor needs `execute-sql` on the database. -- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. - -Default execution rule for user-created writable queries: - -- `is_trusted` must be `0`. -- The actor must have `view-query`. -- The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. - -Implementation: - -- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. -- Emit default `view-query` allows for the owning actor. -- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. -- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. - -For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. - -Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. - -## Writable queries - -Writable user-created queries should be in scope, guarded by `Database.analyze_sql()`. - -The secure rule: a user can create, update, or execute a writable user-created query only if they currently have the corresponding write permissions for every table the SQL can affect. - -`Database.analyze_sql(sql, params=None)` runs the SQL through SQLite's authorizer on an isolated connection and returns a `SQLAnalysis` object containing `SQLTableAccess` rows: - -- `operation`: `read`, `insert`, `update`, or `delete` -- `database`: Datasette database name for `main`, or SQLite schema name where no Datasette mapping exists -- `table`: affected table or view -- `columns`: read/updated columns where SQLite reports them -- `source`: trigger/view/CTE source when SQLite reports one - -Validation flow for user-created queries: - -1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. -2. If analysis raises a SQLite error, reject the query. -3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. -5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. -6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - - `insert` -> `insert-row` - - `update` -> `update-row` - - `delete` -> `delete-row` -7. Include write accesses reported from triggers and views, since those are real side effects. -8. Re-run the same analysis and permission checks when SQL changes through `update_query()` or `POST .../-/update`. -9. Re-run analysis before executing user-created writable queries, so schema or trigger changes cannot leave a previously saved query with stale permission assumptions. - -The user-facing API should not trust a submitted `is_write` value. It should derive `is_write` from analysis. - -Trusted configuration and plugin code can still call `datasette.add_query(..., is_write=True, ...)`. Those are treated as deployment/admin-authored queries. They keep the existing execution model: they require `view-query`, and the default `view-query` hook should preserve current default-open behavior for trusted writable queries while still respecting `--default-deny`. - -Fail closed cases for user-created writable queries: - -- Analysis fails. -- Analysis reports any write operation that cannot be mapped to a Datasette table resource. -- Analysis reports writes outside the target database. -- The actor lacks any required table write permission. -- `is_trusted=1` is requested through the user-facing API. - -This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. - -## HTTP API sketch - -JSON endpoints should follow Datasette's existing write API style: use `POST` plus action paths such as `/-/insert`, `/-/update`, and `/-/delete`, not HTTP `PATCH` or `DELETE`. - -Endpoints: - -- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/insert` creates a query. -- `GET /{database}/{query}/-/definition` returns one query definition without executing it. -- `POST /{database}/{query}/-/update` updates one query. -- `POST /{database}/{query}/-/delete` deletes one query. - -Create request: - -```json -{ - "query": { - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "parameters": ["region"] - } -} -``` - -Successful create returns `201` and the created query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "is_trusted": false, - "parameters": ["region"] - } -} -``` - -Update request, imitating `RowUpdateView`: - -```json -{ - "update": { - "title": "Top customers by revenue", - "is_private": false - }, - "return": true -} -``` - -Successful update returns `{"ok": true}` by default. With `"return": true`, return the updated query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers by revenue", - "is_private": false, - "is_trusted": false - } -} -``` - -Delete request: - -```http -POST /{database}/{query}/-/delete -Content-Type: application/json -``` - -Successful delete returns: - -```json -{ - "ok": true -} -``` - -Validation: - -- Update bodies must be dictionaries containing an `update` dictionary, with optional `return`; invalid keys return `{"ok": false, "errors": [...]}`. -- Validate route-safe query names. -- Reject names that collide with a table or view in the same database, since table routes currently win over query routes. -- Analyze user-created SQL with `Database.analyze_sql()`. -- Use `validate_sql_select(sql)` as the read-only fast path when analysis shows only reads, but do not require it for writable queries that pass analysis and permission checks. -- Reject magic parameters such as `:_actor_id`, `:_cookie_*`, and `:_header_*` for user-created queries. -- Reject client-supplied `is_write`; derive it from analysis. -- Reject writable-only success/error fields for read-only queries. - -## Python API sketch - -Add methods on `Datasette`: - -```python -await datasette.add_query( - database, - name, - sql, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -) - -await datasette.update_query( - database, - name, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) - -await datasette.remove_query(database, name, source=None) - -await datasette.get_query(database, name) -await datasette.list_queries( - database, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -) -``` - -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. - -`update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": - -```python -await datasette.update_query( - "fixtures", - "top_customers", - on_success_redirect=None, -) -``` - -For column-backed fields, `None` should write SQL `NULL`. For option fields, `None` should remove that key from the JSON object so `get_query()` returns `None`; omitting the field should leave the existing option unchanged. - -Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. - -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. - -## Query page save UI - -On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. - -The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. - -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. - -## Dedicated create query UI - -Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. - -This page should require `execute-sql` and `insert-query` to access. It should provide a SQL editor and a mode control: - -- Read-only -- Writable - -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. - -Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: - -- detected operation -- database and table -- required permission -- whether the actor has that permission -- source, when the operation comes from a trigger or view - -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. - -The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. - -## Test plan - -- Internal schema creates `queries`. -- Query parameters are stored in the `queries.parameters` text column as a JSON array of names. -- Config `queries:` blocks import into internal tables. -- Legacy string query definitions normalize to SQL rows. -- The old `canned_queries()` hook is no longer called by core. -- `QueryResource.resources_sql()` returns rows from `queries`. -- Database page and `/-/jump` list queries from the internal DB. -- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. -- Private query is only visible to its owner, even when a broader `view-query` rule applies. -- Non-trusted read-only query requires `execute-sql` to execute. -- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. -- Config queries default to trusted and can opt out with `is_trusted: false`. -- User API rejects client-supplied `is_trusted`. -- User-created query requires both `execute-sql` and `insert-query`. -- User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. -- `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. -- User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be trusted through the user API. -- Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. -- Query delete uses `POST /{database}/{query}/-/delete`. -- There are no `PATCH` or HTTP `DELETE` routes for query management. -- `datasette.update_query(..., field=None)` writes `NULL` for column-backed fields and removes JSON keys for option fields, while omitted fields are left unchanged. -- Owner gets default `update-query` and `delete-query` for their own user-created rows. -- Admin can manage other users' queries with `update-query` and `delete-query`. -- User API rejects magic parameters. -- User API rejects writable queries if analysis fails, reports writes outside the target database, or reports writes the actor is not allowed to perform. -- Trusted config/plugin writable queries still execute through `view-query`. -- Trusted config/plugin writable queries are not default-allowed under `--default-deny`. -- Persisted internal DB does not expose queries for detached databases. From 24887004cffd52fe801ecd73da78e13b246ddede Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:51:57 -0700 Subject: [PATCH 1772/1866] Rename insert-query to store-query Also queries/insert to queries/store Refs https://github.com/simonw/datasette/pull/2741#issuecomment-4549103663 --- datasette/app.py | 6 ++--- datasette/default_actions.py | 6 ++--- datasette/templates/query_create.html | 2 +- datasette/views/database.py | 22 +++++++-------- docs/authentication.rst | 7 ++--- docs/json_api.rst | 5 ++-- tests/test_queries.py | 39 +++++++++++++++------------ 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8936b099..42a2d27d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -54,9 +54,9 @@ from .views.database import ( QueryDeleteView, QueryDefinitionView, GlobalQueryListView, - QueryInsertView, QueryListView, QueryParametersView, + QueryStoreView, QueryUpdateView, ) from .views.index import IndexView @@ -2824,8 +2824,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( - QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/insert$", + QueryStoreView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/store$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6a1f77b8..0f4c25fa 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -62,9 +62,9 @@ def register_actions(): resource_class=DatabaseResource, ), Action( - name="insert-query", - abbr="iq", - description="Create saved queries", + name="store-query", + abbr="sq", + description="Create stored queries", resource_class=DatabaseResource, also_requires="execute-sql", ), diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index cb14ada4..f5dadbff 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -156,7 +156,7 @@ form.sql .query-create-sql textarea#sql-editor {

    Create query

    -
    +

    {{ urls.database(database) }}/

    diff --git a/datasette/views/database.py b/datasette/views/database.py index d40d69d1..900b94ba 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1419,7 +1419,7 @@ class QueryCreateView(BaseView): actor=request.actor, ) await self.ds.ensure_permission( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ) @@ -1440,11 +1440,11 @@ class QueryCreateAnalyzeView(BaseView): ): return _block_framing(_error(["Permission denied: need execute-sql"], 403)) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need insert-query"], 403)) + return _block_framing(_error(["Permission denied: need store-query"], 403)) invalid_keys = set(request.args) - {"sql"} if invalid_keys: @@ -1462,8 +1462,8 @@ class QueryCreateAnalyzeView(BaseView): ) -class QueryInsertView(QueryCreateView): - name = "query-insert" +class QueryStoreView(QueryCreateView): + name = "query-store" async def _error_response(self, request, db, query_data, message, status): message = _query_create_form_error_message(message) @@ -1488,11 +1488,11 @@ class QueryInsertView(QueryCreateView): ): return _error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need insert-query"], 403) + return _error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -1961,8 +1961,8 @@ class QueryView(View): resource=DatabaseResource(database=database), actor=request.actor, ) - allow_insert_query = await datasette.allowed( - action="insert-query", + allow_store_query = await datasette.allowed( + action="store-query", resource=DatabaseResource(database=database), actor=request.actor, ) @@ -2020,13 +2020,13 @@ class QueryView(View): if ( not canned_query and allow_execute_sql - and allow_insert_query + and allow_store_query and is_validated_sql and ":_" not in sql ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/insert?" + + "/-/queries/store?" + urlencode({"sql": sql}) ) diff --git a/docs/authentication.rst b/docs/authentication.rst index 453aaa19..184fec5e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1293,11 +1293,12 @@ Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fi ``query`` is the name of the query (string) .. _actions_insert_query: +.. _actions_store_query: -insert-query ------------- +store-query +----------- -Actor is allowed to create saved queries in a database. +Actor is allowed to create stored queries in a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/json_api.rst b/docs/json_api.rst index dd54c459..1a6c7021 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -518,14 +518,15 @@ Listing saved queries Creating saved queries in the UI ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries/-/create`` provides a form for creating saved queries. +``GET //-/queries/store`` provides a form for creating stored queries. +.. _QueryStoreView: .. _QueryInsertView: Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: diff --git a/tests/test_queries.py b/tests/test_queries.py index 26a0748c..5d4da9bb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -470,7 +470,7 @@ async def test_query_actions_are_registered(): await ds.invoke_startup() assert ds.get_action("execute-write-sql").resource_class is DatabaseResource - assert ds.get_action("insert-query").resource_class is DatabaseResource + assert ds.get_action("store-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -537,15 +537,15 @@ async def test_analyze_write_query_rejects_writes_to_attached_databases(): @pytest.mark.asyncio -async def test_query_insert_api_creates_read_only_query(): +async def test_query_store_api_creates_read_only_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True - db = ds.add_memory_database("query_insert_api", name="data") + db = ds.add_memory_database("query_store_api", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -860,7 +860,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_is_trusted(): +async def test_query_store_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -870,7 +870,7 @@ async def test_query_insert_api_rejects_is_trusted(): "permissions": { "view-database": {"id": "writer"}, "execute-sql": {"id": "writer"}, - "insert-query": {"id": "writer"}, + "store-query": {"id": "writer"}, } } } @@ -880,7 +880,7 @@ async def test_query_insert_api_rejects_is_trusted(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "writer"}, json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) @@ -890,7 +890,7 @@ async def test_query_insert_api_rejects_is_trusted(): @pytest.mark.asyncio -async def test_query_insert_api_creates_writable_query(): +async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True db = ds.add_memory_database("query_write_api", name="data") @@ -898,7 +898,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -962,14 +962,14 @@ async def test_query_update_and_delete_api(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_magic_parameters(): +async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True ds.add_memory_database("query_magic_api", name="data") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -987,15 +987,19 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( - "/data/-/queries/insert?sql=select+*+from+dogs", + "/data/-/queries/store?sql=select+*+from+dogs", actor={"id": "root"}, ) write_create_response = await ds.client.get( - "/data/-/queries/insert?sql=insert+into+dogs+(name)+values+('Cleo')", + "/data/-/queries/store?sql=insert+into+dogs+(name)+values+('Cleo')", actor={"id": "root"}, ) blank_create_response = await ds.client.get( - "/data/-/queries/insert", + "/data/-/queries/store", + actor={"id": "root"}, + ) + old_insert_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", actor={"id": "root"}, ) old_create_response = await ds.client.get( @@ -1075,7 +1079,8 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) assert query_response.status_code == 200 assert "Save this query" in query_response.text - assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text + assert "/data/-/queries/store?sql=select+%2A+from+dogs" in query_response.text + assert old_insert_response.status_code == 404 assert old_create_response.status_code == 404 @@ -1153,7 +1158,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", @@ -1176,7 +1181,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): assert 'name="is_private" value="1" checked' in response.text public_response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", From 0cadd071871ef0b33e4ce3a23e316a104b3137c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:53:31 -0700 Subject: [PATCH 1773/1866] No need to document QueryCreateAnalyzeView --- tests/test_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 396ba1a2..0d0ef1e1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -66,7 +66,14 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) + view_labels.update( + ( + "PatternPortfolioView", + "AuthTokenView", + "ApiExplorerView", + "QueryCreateAnalyzeView", + ) + ) return view_labels From 4bf1c4b065fef64676abf5eabd04ff35e07188c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:54:35 -0700 Subject: [PATCH 1774/1866] Rename canned queries to queries/stored queries in docs --- datasette/default_actions.py | 4 +- datasette/hookspecs.py | 4 +- datasette/resources.py | 2 +- datasette/views/database.py | 24 ++++----- datasette/views/table.py | 4 +- docs/authentication.rst | 16 +++--- docs/configuration.rst | 10 ++-- docs/custom_templates.rst | 8 +-- docs/internals.rst | 12 ++--- docs/introspection.rst | 2 +- docs/json_api.rst | 32 ++++++------ docs/pages.rst | 4 +- docs/plugin_hooks.rst | 16 +++--- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 95 ++++++++++++++++++++++++++---------- tests/test_html.py | 6 +-- tests/test_permissions.py | 4 +- 17 files changed, 144 insertions(+), 101 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 0f4c25fa..2f78570b 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -121,13 +121,13 @@ def register_actions(): Action( name="update-query", abbr="uq", - description="Update saved queries", + description="Update stored queries", resource_class=QueryResource, ), Action( name="delete-query", abbr="dq", - description="Delete saved queries", + description="Delete stored queries", resource_class=QueryResource, ), ) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a4067eaa..22da02a4 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -174,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and canned query actions menu""" + """Links for the query and stored query actions menu""" @hookspec @@ -229,7 +229,7 @@ def top_query(datasette, request, database, sql): @hookspec def top_canned_query(datasette, request, database, query_name): - """HTML to include at the top of the canned query page""" + """HTML to include at the top of the stored query page""" @hookspec diff --git a/datasette/resources.py b/datasette/resources.py index 91a46d36..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A saved query in a database.""" + """A stored query in a database.""" name = "query" parent_class = DatabaseResource diff --git a/datasette/views/database.py b/datasette/views/database.py index 900b94ba..f30d3815 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -222,11 +222,11 @@ class DatabaseContext(Context): tables: list = field(metadata={"help": "List of table objects in the database"}) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) - queries: list = field(metadata={"help": "List of canned query objects"}) + queries: list = field(metadata={"help": "List of stored query objects"}) queries_more: bool = field( - metadata={"help": "Boolean indicating if more saved queries are available"} + metadata={"help": "Boolean indicating if more stored queries are available"} ) - queries_count: int = field(metadata={"help": "Count of visible saved queries"}) + queries_count: int = field(metadata={"help": "Count of visible stored queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -272,7 +272,7 @@ class QueryContext(Context): metadata={"help": "The SQL query object containing the `sql` string"} ) canned_query: str = field( - metadata={"help": "The name of the canned query if this is a canned query"} + metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -282,11 +282,11 @@ class QueryContext(Context): # ) canned_query_write: bool = field( metadata={ - "help": "Boolean indicating if this is a canned query that allows writes" + "help": "Boolean indicating if this is a stored query that allows writes" } ) metadata: dict = field( - metadata={"help": "Metadata about the database or the canned query"} + metadata={"help": "Metadata about the database or the stored query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -315,7 +315,7 @@ class QueryContext(Context): metadata={"help": "Dictionary of parameter names/values"} ) edit_sql_url: str = field( - metadata={"help": "URL to edit the SQL for a canned query"} + metadata={"help": "URL to edit the SQL for a stored query"} ) display_rows: list = field(metadata={"help": "List of result rows to display"}) columns: list = field(metadata={"help": "List of column names"}) @@ -1623,7 +1623,7 @@ class QueryView(View): db = await datasette.resolve_database(request) - # We must be a canned query + # We must be a stored query table_found = False try: await datasette.resolve_table(request) @@ -1742,14 +1742,14 @@ class QueryView(View): # Create lookup dict for quick access allowed_dict = {r.child: r for r in allowed_tables_page.resources} - # Are we a canned query? + # Are we a stored query? canned_query = None canned_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( table_not_found.database_name, table_not_found.table, request.actor ) @@ -1759,7 +1759,7 @@ class QueryView(View): private = False if canned_query: - # Respect canned query permissions + # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", @@ -1823,7 +1823,7 @@ class QueryView(View): # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: - # Canned queries can run magic parameters + # Stored queries can run magic parameters params_for_query = MagicParameters(sql, params, request, datasette) await params_for_query.execute_params() results = await datasette.execute( diff --git a/datasette/views/table.py b/datasette/views/table.py index 7027bb10..7b1a5a82 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -963,11 +963,11 @@ async def table_view_traced(datasette, request): try: resolved = await datasette.resolve_table(request) except TableNotFound as not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( not_found.database_name, not_found.table, request.actor ) - # If this is a canned query, not a table, then dispatch to QueryView instead + # If this is a stored query, not a table, then dispatch to QueryView instead if canned_query: return await QueryView()(request, datasette) else: diff --git a/docs/authentication.rst b/docs/authentication.rst index 184fec5e..22db41d8 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`canned_queries` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -641,12 +641,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` .. _authentication_permissions_query: -Access to specific canned queries ---------------------------------- +Access to specific queries +-------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. -To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: +To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog config_example(cog, """ @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. +Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted stored query also requires ``execute-sql`` or the relevant write permissions; :ref:`trusted stored queries ` can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1308,7 +1308,7 @@ Actor is allowed to create stored queries in a database. update-query ------------ -Actor is allowed to update a saved query. +Actor is allowed to update a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1320,7 +1320,7 @@ Actor is allowed to update a saved query. delete-query ------------ -Actor is allowed to delete a saved query. +Actor is allowed to delete a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c8c8a67..cf9590b8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -87,6 +87,7 @@ This is equivalent to a ``datasette.yaml`` file containing the following: } .. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference @@ -435,10 +436,10 @@ Here is a simple example: .. _configuration_reference_canned_queries: -Canned queries configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Queries configuration +~~~~~~~~~~~~~~~~~~~~~ -:ref:`Canned queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -483,7 +484,7 @@ Canned queries configuration } .. [[[end]]] -See the :ref:`canned queries documentation ` for more, including how to configure :ref:`writable canned queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: @@ -1211,4 +1212,3 @@ For column types that accept additional configuration, use an object with ``type } } .. [[[end]]] - diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 8cc40f0f..c324fb79 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -29,7 +29,7 @@ The custom SQL template (``/dbname?sql=...``) gets this: -A canned query template (``/dbname/queryname``) gets this: +A stored query template (``/dbname/queryname``) gets this: .. code-block:: html @@ -193,8 +193,8 @@ The lookup rules Datasette uses are as follows:: query-mydatabase.html query.html - Canned query page (/mydatabase/canned-query): - query-mydatabase-canned-query.html + Stored query page (/mydatabase/query-name): + query-mydatabase-query-name.html query-mydatabase.html query.html @@ -230,7 +230,7 @@ will look something like this:: -This example is from the canned query page for a query called "tz" in the +This example is from the stored query page for a query called "tz" in the database called "mydb". The asterisk shows which template was selected - so in this case, Datasette found a template file called ``query-mydb-tz.html`` and used that - but if that template had not been found, it would have tried for diff --git a/docs/internals.rst b/docs/internals.rst index c76de487..084922f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -725,7 +725,7 @@ The builder methods are: - ``allow_all(action)`` - allow an action across all databases and resources - ``allow_database(database, action)`` - allow an action on a specific database -- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`canned query `) within a database +- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`stored query `) within a database Each method returns the ``TokenRestrictions`` instance so calls can be chained. @@ -837,10 +837,10 @@ await .get_resource_metadata(self, database_name, resource_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. Returns metadata keys and values for the specified "resource" as a dictionary. -A "resource" in this context can be a table, view, or canned query. +A "resource" in this context can be a table, view, or stored query. Internally queries the ``metadata_resources`` table inside the :ref:`internal database `. .. _datasette_get_column_metadata: @@ -851,7 +851,7 @@ await .get_column_metadata(self, database_name, resource_name, column_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. ``column_name`` - string The name of the column inside ``resource_name`` to query. @@ -897,7 +897,7 @@ await .set_resource_metadata(self, database_name, resource_name, key, value) ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``key`` - string The metadata entry key to insert (ex ``title``, ``description``, etc.) ``value`` - string @@ -915,7 +915,7 @@ await .set_column_metadata(self, database_name, resource_name, column_name, key, ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``column-name`` - string The column the metadata entry belongs to. ``key`` - string diff --git a/docs/introspection.rst b/docs/introspection.rst index d2eb8efd..7702a4b5 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -149,7 +149,7 @@ Shows currently attached databases. `Databases example /-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns stored query definitions across every database that the actor can view. ``GET //-/queries.json`` returns stored query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: -Creating saved queries in the UI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries in the UI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``GET //-/queries/store`` provides a form for creating stored queries. .. _QueryStoreView: .. _QueryInsertView: -Creating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ ``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. @@ -545,24 +545,24 @@ Executing write SQL .. _QueryDefinitionView: -Getting a saved query definition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Getting a stored query definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET ///-/definition`` returns a saved query definition without executing it. +``GET ///-/definition`` returns a stored query definition without executing it. .. _QueryUpdateView: -Updating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Updating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/update`` updates a saved query using a JSON body with an ``"update"`` object. +``POST ///-/update`` updates a stored query using a JSON body with an ``"update"`` object. .. _QueryDeleteView: -Deleting saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/delete`` deletes a saved query. +``POST ///-/delete`` deletes a stored query. .. _TableInsertView: diff --git a/docs/pages.rst b/docs/pages.rst index 34c851a5..e57c15e6 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -28,7 +28,7 @@ The index page can also be accessed at ``/-/``, useful for if the default index Database ======== -Each database has a page listing the tables, views and canned queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. +Each database has a page listing the tables, views and stored queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. Examples: @@ -68,7 +68,7 @@ This means you can link directly to a query by constructing the following URL: ``/database-name/-/query?sql=SELECT+*+FROM+table_name`` -Each configured :ref:`canned query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. +Each configured :ref:`stored query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. In both cases adding a ``.json`` extension to the URL will return the results as JSON. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b2676b3e..264b473e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -609,7 +609,7 @@ When a request is received, the ``"render"`` callback function is called with ze The SQL query that was executed. ``query_name`` - string or None - If this was the execution of a :ref:`canned query `, the name of that query. + If this was the execution of a :ref:`stored query `, the name of that query. ``database`` - string The name of the database. @@ -1212,7 +1212,7 @@ Examples: `datasette-saved-queries `__ @@ -1635,7 +1635,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. @@ -1828,7 +1828,7 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and stored query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. ``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. @@ -2004,7 +2004,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) The name of the database. ``query_name`` - string or None - The name of the canned query, or ``None`` if this is an arbitrary SQL query. + The name of the stored query, or ``None`` if this is an arbitrary SQL query. ``request`` - :ref:`internals_request` The current HTTP request. @@ -2015,7 +2015,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) ``params`` - dictionary The parameters passed to the SQL query, if any. -Populates a "Query actions" menu on the canned query and arbitrary SQL query pages. +Populates a "Query actions" menu on the stored query and arbitrary SQL query pages. This example adds a new query action linking to a page for explaining a query: @@ -2294,9 +2294,9 @@ top_canned_query(datasette, request, database, query_name) The name of the database. ``query_name`` - string - The name of the canned query. + The name of the stored query. -Returns HTML to be displayed at the top of the canned query page. +Returns HTML to be displayed at the top of the stored query page. .. _plugin_event_tracking: diff --git a/docs/spatialite.rst b/docs/spatialite.rst index c93c1e00..1999ab78 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`canned_queries` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 7c3cd4ac..d60656e3 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -68,10 +68,10 @@ You can also use the `sqlite-utils `__ tool .. _canned_queries: -Canned queries --------------- +Queries +------- -As an alternative to adding views to your database, you can define canned queries inside your ``datasette.yaml`` file. Here's an example: +As an alternative to adding views to your database, you can define named queries inside your ``datasette.yaml`` file. Here's an example: .. [[[cog from metadata_doc import config_example, config_example @@ -120,24 +120,67 @@ Then run Datasette like this:: datasette sf-trees.db -m metadata.json -Each canned query will be listed on the database index page, and will also get its own URL at:: +Each configured query will be listed on the database index page, and will also get its own URL at:: - /database-name/canned-query-name + /database-name/query-name For the above example, that URL would be:: /sf-trees/just_species -You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). +You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). + +.. _stored_queries: +.. _saved_queries: + +Stored queries +~~~~~~~~~~~~~~ + +Datasette stores both configured queries and user-created queries in the ``queries`` table in the :ref:`internal database `. Configured queries come from the ``queries`` section of ``datasette.yaml``. User-created stored queries can be created from the SQL query page by actors with the :ref:`actions_store_query` and :ref:`actions_execute_sql` permissions. Writable stored queries also require the permissions needed for the writes they perform. + +Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. + +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. + +.. _trusted_stored_queries: +.. _trusted_saved_queries: + +Trusted stored queries +++++++++++++++++++++++ + +A trusted stored query can execute with ``view-query`` permission alone. It skips the additional ``execute-sql`` and write permission checks that are applied to untrusted stored queries. + +Trusted stored queries should only be used for SQL that has been reviewed by someone trusted to configure the Datasette instance. For that reason, trusted stored queries can only be added using configuration. Users cannot create trusted stored queries through the web interface or the stored query JSON API. + +Queries defined in ``datasette.yaml`` are trusted by default: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + +You can opt out of this behavior for a configured query using ``is_trusted: false``: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + is_trusted: false .. _canned_queries_named_parameters: -Canned query parameters -~~~~~~~~~~~~~~~~~~~~~~~ +Query parameters +~~~~~~~~~~~~~~~~ -Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement. +Configured queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the query page or by adding them to the URL. This means configured queries can be used to create custom JSON APIs based on a carefully designed SQL statement. -Here's an example of a canned query with a named parameter: +Here's an example of a configured query with a named parameter: .. code-block:: sql @@ -147,7 +190,7 @@ Here's an example of a canned query with a named parameter: where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query configuration looks like this: +The query configuration looks like this: .. [[[cog @@ -204,7 +247,7 @@ In the canned query configuration looks like this: Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. -You can try this canned query out here: +You can try this query out here: https://latest.datasette.io/fixtures/neighborhood_search?text=town In this example the ``:text`` named parameter is automatically extracted from the query using a regular expression. @@ -272,15 +315,15 @@ You can alternatively provide an explicit list of named parameters using the ``" .. _canned_queries_options: -Additional canned query options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Additional query options +~~~~~~~~~~~~~~~~~~~~~~~~ -Additional options can be specified for canned queries in the YAML or JSON configuration. +Additional options can be specified for configured queries in the YAML or JSON configuration. hide_sql ++++++++ -Canned queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. +Configured queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. Add the ``"hide_sql": true`` option to hide the SQL query by default. @@ -289,7 +332,7 @@ fragment Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. -You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key. +You can set a default fragment hash that will be included in the link to the query from the database index page using the ``"fragment"`` key. This example demonstrates both ``fragment`` and ``hide_sql``: @@ -348,12 +391,12 @@ This example demonstrates both ``fragment`` and ``hide_sql``: .. _canned_queries_writable: -Writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~ +Writable queries +~~~~~~~~~~~~~~~~ -Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. +Configured queries are read-only by default. You can use the ``"write": true`` key to indicate that a query can write to the database. -See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key. +See :ref:`authentication_permissions_query` for details on how to add permission checks to queries, using the ``"allow"`` key. .. [[[cog config_example(cog, { @@ -488,7 +531,7 @@ Magic parameters Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string. -These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. +These magic parameters are only supported for configured queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. Available magic parameters are: @@ -580,12 +623,12 @@ Additional custom magic parameters can be added by plugins using the :ref:`plugi .. _canned_queries_json_api: -JSON API for writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +JSON API for writable queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Writable canned queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. +Writable queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. -To submit JSON to a writable canned query, encode key/value parameters as a JSON document:: +To submit JSON to a writable query, encode key/value parameters as a JSON document:: POST /mydatabase/add_message diff --git a/tests/test_html.py b/tests/test_html.py index 9e460da1..8edb9f6e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -154,7 +154,7 @@ async def test_database_page(ds_client): ("/fixtures/simple_view", "simple_view"), ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) - # And a list of canned queries + # And a list of stored queries queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ @@ -701,7 +701,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): - # For a canned query the show/hide should NOT have a hidden SQL field + # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") soup = Soup(response.content, "html.parser") @@ -1106,7 +1106,7 @@ async def test_trace_correctly_escaped(ds_client): "/fixtures/-/query?sql=select+*+from+facetable", "http://localhost/fixtures/-/query.json?sql=select+*+from+facetable", ), - # Canned query page + # Stored query page ( "/fixtures/neighborhood_search?text=town", "http://localhost/fixtures/neighborhood_search.json?text=town", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index eb6cee9f..0e38c876 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -890,7 +890,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # view-query on canned query, wrong actor + # view-query on stored query, wrong actor PermConfigTestCase( config={ "databases": { @@ -909,7 +909,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "q1"), expected_result=False, ), - # view-query on canned query, right actor + # view-query on stored query, right actor PermConfigTestCase( config={ "databases": { From b1029acc68626c2fddf7b678adc3339be0fce6e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:05:41 -0700 Subject: [PATCH 1775/1866] top_canned_query is now top_stored_query, closes #2747 --- datasette/hookspecs.py | 2 +- datasette/templates/query.html | 2 +- datasette/views/database.py | 8 ++++---- docs/changelog.rst | 1 + docs/plugin_hooks.rst | 4 ++-- tests/test_plugins.py | 10 ++++++---- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 22da02a4..dcd502af 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -228,7 +228,7 @@ def top_query(datasette, request, database, sql): @hookspec -def top_canned_query(datasette, request, database, query_name): +def top_stored_query(datasette, request, database, query_name): """HTML to include at the top of the stored query page""" diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 785b05af..3f03424a 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -33,7 +33,7 @@ {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} +{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index f30d3815..def3c530 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -339,8 +339,8 @@ class QueryContext(Context): top_query: callable = field( metadata={"help": "Callable to render the top_query slot"} ) - top_canned_query: callable = field( - metadata={"help": "Callable to render the top_canned_query slot"} + top_stored_query: callable = field( + metadata={"help": "Callable to render the top_stored_query slot"} ) query_actions: callable = field( metadata={ @@ -2095,8 +2095,8 @@ class QueryView(View): top_query=make_slot_function( "top_query", datasette, request, database=database, sql=sql ), - top_canned_query=make_slot_function( - "top_canned_query", + top_stored_query=make_slot_function( + "top_stored_query", datasette, request, database=database, diff --git a/docs/changelog.rst b/docs/changelog.rst index dfb2a736..300ac02f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) +- The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) .. _v1_0_a30: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 264b473e..4737ca03 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -2279,9 +2279,9 @@ top_query(datasette, request, database, sql) Returns HTML to be displayed at the top of the query results page. -.. _plugin_hook_top_canned_query: +.. _plugin_hook_top_stored_query: -top_canned_query(datasette, request, database, query_name) +top_stored_query(datasette, request, database, query_name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``datasette`` - :ref:`internals_datasette` diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f7adbd66..32276437 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1486,8 +1486,10 @@ class SlotPlugin: return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) @hookimpl - def top_canned_query(self, request, database, query_name): - return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + def top_stored_query(self, request, database, query_name): + return "Xtop_stored_query:{}:{}:{}".format( + database, query_name, request.args["z"] + ) @pytest.mark.asyncio @@ -1548,12 +1550,12 @@ async def test_hook_top_query(ds_client): @pytest.mark.asyncio -async def test_hook_top_canned_query(ds_client): +async def test_hook_top_stored_query(ds_client): try: pm.register(SlotPlugin(), name="SlotPlugin") response = await ds_client.get("/fixtures/magic_parameters?z=xyz") assert response.status_code == 200 - assert "Xtop_query:fixtures:magic_parameters:xyz" in response.text + assert "Xtop_stored_query:fixtures:magic_parameters:xyz" in response.text finally: pm.unregister(name="SlotPlugin") From 2f73869c09962e320e5f40f4691df70618cd052e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:09:48 -0700 Subject: [PATCH 1776/1866] Document that canned_queries() has been removed --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 300ac02f..674ff5b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. .. _v1_0_a30: From 56b14f37d547e03ba902516ac9ae13ef52765f77 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:16:18 -0700 Subject: [PATCH 1777/1866] The stored queries do not live in that DB --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 22db41d8..86df7f04 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1298,7 +1298,7 @@ Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/f store-query ----------- -Actor is allowed to create stored queries in a database. +Actor is allowed to create stored queries against a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) From 02a1468f1b3c8c14fb80037686b43de856e49c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:17:51 -0700 Subject: [PATCH 1778/1866] Renamed canned queries to queries / stored queries in docs And a few renames in code and YAML as well. --- .github/workflows/deploy-latest.yml | 33 +- datasette/app.py | 7 - datasette/facets.py | 2 +- datasette/static/app.css | 2 +- datasette/templates/query.html | 18 +- datasette/views/database.py | 92 +++--- datasette/views/table.py | 6 +- docs/authentication.rst | 10 +- docs/changelog.rst | 23 +- docs/configuration.rst | 6 +- docs/plugin_hooks.rst | 12 +- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 12 +- docs/upgrade-1.0a20.md | 6 +- tests/test_canned_queries.py | 473 ---------------------------- tests/test_html.py | 12 +- tests/test_jump.py | 4 +- 17 files changed, 115 insertions(+), 605 deletions(-) delete mode 100644 tests/test_canned_queries.py diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7d8dd37d..166d33d0 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py <This query cannot be executed because the database is immutable.

    {% endif %} -

    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

    +

    {{ metadata.title or database }}{% if stored_query and not metadata.title %}: {{ stored_query }}{% endif %}{% if private %} 🔒{% endif %}

    {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} +{% if stored_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    @@ -52,7 +52,7 @@ 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} - {% if not canned_query %} + {% if not stored_query %} @@ -64,10 +64,10 @@ {% include "_sql_parameters.html" %}

    {% if not hide_sql %}{% endif %} - + {{ show_hide_hidden }} {% if save_query_url %}Save this query{% endif %} - {% if canned_query and edit_sql_url %}Edit SQL{% endif %} + {% if stored_query and edit_sql_url %}Edit SQL{% endif %}

    @@ -90,7 +90,7 @@
    Required permission
    {% else %} - {% if not canned_query_write and not error %} + {% if not stored_query_write and not error %}

    0 results

    {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index def3c530..c36476f6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -100,12 +100,12 @@ class DatabaseView(View): limit=5, include_private=True, ) - canned_queries = queries_page["queries"] + stored_queries = queries_page["queries"] queries_more = queries_page["has_more"] queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more - else len(canned_queries) + else len(stored_queries) ) async def database_actions(): @@ -137,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": canned_queries, + "queries": stored_queries, "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -172,7 +172,7 @@ class DatabaseView(View): tables=tables, hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, - queries=canned_queries, + queries=stored_queries, queries_more=queries_more, queries_count=queries_count, allow_execute_sql=allow_execute_sql, @@ -271,7 +271,7 @@ class QueryContext(Context): query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) - canned_query: str = field( + stored_query: str = field( metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( @@ -280,7 +280,7 @@ class QueryContext(Context): # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_query_write: bool = field( + stored_query_write: bool = field( metadata={ "help": "Boolean indicating if this is a stored query that allows writes" } @@ -1629,10 +1629,10 @@ class QueryView(View): await datasette.resolve_table(request) table_found = True except TableNotFound as table_not_found: - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise if table_found: # That should not have happened @@ -1640,13 +1640,13 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=canned_query["name"]), + resource=QueryResource(database=db.name, query=stored_query["name"]), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) # If database is immutable, return an error @@ -1674,19 +1674,19 @@ class QueryView(View): or params.get("_json") ) params_for_query = MagicParameters( - canned_query["sql"], params, request, datasette + stored_query["sql"], params, request, datasette ) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - canned_query["sql"], params_for_query, request=request + stored_query["sql"], params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = canned_query.get("on_success_message_sql") + on_success_message_sql = stored_query.get("on_success_message_sql") if on_success_message_sql: try: message_result = ( @@ -1698,18 +1698,18 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = canned_query.get( + message = stored_query.get( "on_success_message" ) or "Query executed, {} row{} affected".format( cursor.rowcount, "" if cursor.rowcount == 1 else "s" ) - redirect_url = canned_query.get("on_success_redirect") + redirect_url = stored_query.get("on_success_redirect") ok = True except Exception as ex: - message = canned_query.get("on_error_message") or str(ex) + message = stored_query.get("on_error_message") or str(ex) message_type = datasette.ERROR - redirect_url = canned_query.get("on_error_redirect") + redirect_url = stored_query.get("on_error_redirect") ok = False if should_return_json: return Response.json( @@ -1743,33 +1743,33 @@ class QueryView(View): allowed_dict = {r.child: r for r in allowed_tables_page.resources} # Are we a stored query? - canned_query = None - canned_query_write = False + stored_query = None + stored_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise - canned_query_write = bool(canned_query.get("write")) + stored_query_write = bool(stored_query.get("write")) private = False - if canned_query: + if stored_query: # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=canned_query["name"]), + resource=QueryResource(database=database, query=stored_query["name"]), ) if not visible: raise Forbidden("You do not have permission to view this query") - if not canned_query_write: + if not stored_query_write: await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) else: @@ -1783,15 +1783,15 @@ class QueryView(View): params = {key: request.args.get(key) for key in request.args} sql = None - if canned_query: - sql = canned_query["sql"] + if stored_query: + sql = stored_query["sql"] elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if canned_query and canned_query.get("params"): - named_parameters = canned_query["params"] + if stored_query and stored_query.get("params"): + named_parameters = stored_query["params"] if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -1817,9 +1817,9 @@ class QueryView(View): params_for_query = params - if sql and not canned_query_write: + if sql and not stored_query_write: try: - if not canned_query: + if not stored_query: # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: @@ -1879,7 +1879,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, database=database, table=None, request=request, @@ -1911,10 +1911,10 @@ class QueryView(View): elif format_ == "html": headers = {} templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: + if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", ) environment = datasette.get_jinja_environment(request) @@ -1932,8 +1932,8 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) - if canned_query: - metadata = dict(canned_query) + if stored_query: + metadata = dict(stored_query) metadata.pop("source", None) renderers = {} @@ -1968,7 +1968,7 @@ class QueryView(View): ) show_hide_hidden = "" - if canned_query and canned_query.get("hide_sql"): + if stored_query and stored_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -2018,7 +2018,7 @@ class QueryView(View): ) save_query_url = None if ( - not canned_query + not stored_query and allow_execute_sql and allow_store_query and is_validated_sql @@ -2036,7 +2036,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, request=request, sql=sql, params=params, @@ -2056,15 +2056,15 @@ class QueryView(View): "sql": sql, "params": params, }, - canned_query=canned_query["name"] if canned_query else None, + stored_query=stored_query["name"] if stored_query else None, private=private, - canned_query_write=canned_query_write, + stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, error=query_error, hide_sql=hide_sql, show_hide_link=datasette.urls.path(show_hide_link), show_hide_text=show_hide_text, - editable=not canned_query, + editable=not stored_query, allow_execute_sql=allow_execute_sql, save_query_url=save_query_url, tables=await get_tables(datasette, request, db, allowed_dict), @@ -2100,7 +2100,7 @@ class QueryView(View): datasette, request, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/table.py b/datasette/views/table.py index 7b1a5a82..da69c6b5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -964,11 +964,11 @@ async def table_view_traced(datasette, request): resolved = await datasette.resolve_table(request) except TableNotFound as not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - not_found.database_name, not_found.table, request.actor + stored_query = await datasette.get_query( + not_found.database_name, not_found.table ) # If this is a stored query, not a table, then dispatch to QueryView instead - if canned_query: + if stored_query: return await QueryView()(request, datasette) else: raise diff --git a/docs/authentication.rst b/docs/authentication.rst index 86df7f04..cec47f97 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -121,7 +121,7 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`queries ` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json @@ -644,7 +644,7 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific queries -------------------------- -:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: @@ -1020,7 +1020,7 @@ You can also restrict permissions such that they can only be used within specifi The resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database. -Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: +Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: datasette create-token root --resource mydatabase mytable insert-row diff --git a/docs/changelog.rst b/docs/changelog.rst index 674ff5b3..d15dec50 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,8 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) -- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to manage stored queries instead. +- The ``datasette.get_canned_query()`` and ``datasette.get_canned_queries()`` methods have been removed. Plugins can use ``datasette.get_query()`` and ``datasette.list_queries()`` instead. .. _v1_0_a30: @@ -658,7 +659,7 @@ For more information and workarounds, read `the security advisory `` in a ``
    `` element, to help with advanced CSS customization. (:issue:`1446`) - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`) - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`) -- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`) +- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`queries_options`. (:issue:`1422`) - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`) - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`) - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`) @@ -1426,7 +1427,7 @@ See also `Datasette 0.50: The annotated release notes `__. -- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (:issue:`880`) +- Writable canned queries now expose a JSON API, see :ref:`queries_json_api`. (:issue:`880`) - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (:issue:`944`) - ``register_output_renderer()`` render functions can now return a ``Response``. (:issue:`953`) - New ``--upgrade`` option for ``datasette install``. (:issue:`945`) @@ -1518,7 +1519,7 @@ Magic parameters for canned queries, a log out feature, improved plugin document Magic parameters for canned queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Canned queries now support :ref:`canned_queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: +Canned queries now support :ref:`queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: insert into logs (user_id, timestamp) @@ -1549,7 +1550,7 @@ New plugin hooks - :ref:`plugin_hook_register_magic_parameters` can be used to define new types of magic canned query parameters. - :ref:`plugin_hook_startup` can run custom code when Datasette first starts up. `datasette-init `__ is a new plugin that uses this hook to create database tables and views on startup if they have not yet been created. (:issue:`834`) -- :ref:`plugin_hook_canned_queries` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) +- ``canned_queries()`` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) - :ref:`plugin_hook_forbidden` is a hook for customizing how Datasette responds to 403 forbidden errors. (:issue:`812`) Smaller changes @@ -1624,7 +1625,7 @@ A new debug page at ``/-/permissions`` shows recent permission checks, to help a Writable canned queries ~~~~~~~~~~~~~~~~~~~~~~~ -Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. +Datasette's :ref:`queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (:issue:`800`): @@ -1643,7 +1644,7 @@ Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 intr } } -See :ref:`canned_queries_writable` for more details. +See :ref:`queries_writable` for more details. Flash messages ~~~~~~~~~~~~~~ @@ -1698,7 +1699,7 @@ Smaller changes - New ``request.cookies`` property. - ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1 - ``request.post_vars()`` method no longer discards empty values. -- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (:issue:`797`) +- New "params" canned query key for explicitly setting named parameters, see :ref:`queries_named_parameters`. (:issue:`797`) - ``request.args`` is now a :ref:`MultiParams ` object. - Fixed a bug with the ``datasette plugins`` command. (:issue:`802`) - Nicer pattern for using ``make_app_client()`` in tests. (:issue:`395`) @@ -1732,7 +1733,7 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`) * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`) * New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`table_configuration_size`. (:issue:`751`) -* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_options`. (:issue:`706`) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`queries_options`. (:issue:`706`) * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`) * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`) @@ -2249,7 +2250,7 @@ A number of small new features: - Documentation for :ref:`datasette publish and datasette package `, closes `#337 `_ - Fixed compatibility with Python 3.7 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett] -- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ +- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ - New ``force_https_on`` config option, fixes ``https://`` API URLs when deploying to Zeit Now - closes `#333 `_ - ``?_json_infinity=1`` query string argument for handling Infinity/-Infinity values in JSON, closes `#332 `_ - URLs displayed in the results of custom SQL queries are now URLified, closes `#298 `_ diff --git a/docs/configuration.rst b/docs/configuration.rst index cf9590b8..9213a640 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -434,12 +434,12 @@ Here is a simple example: :ref:`authentication_permissions_config` has the full details. -.. _configuration_reference_canned_queries: +.. _configuration_reference_queries: Queries configuration ~~~~~~~~~~~~~~~~~~~~~ -:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -484,7 +484,7 @@ Queries configuration } .. [[[end]]] -See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4737ca03..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1207,16 +1207,6 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ -.. _plugin_hook_canned_queries: - -canned_queries(datasette, database, actor) ------------------------------------------- - -This hook has been removed. Plugins that need to add stored queries should use -the :ref:`plugin_hook_startup` hook and call ``await datasette.add_query(...)``. - -Example: `datasette-saved-queries `__ - .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) @@ -1635,7 +1625,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 1999ab78..bea679aa 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index d60656e3..f593a534 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -66,7 +66,7 @@ You can also use the `sqlite-utils `__ tool sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" -.. _canned_queries: +.. _queries: Queries ------- @@ -173,7 +173,7 @@ You can opt out of this behavior for a configured query using ``is_trusted: fals sql: select * from report is_trusted: false -.. _canned_queries_named_parameters: +.. _queries_named_parameters: Query parameters ~~~~~~~~~~~~~~~~ @@ -313,7 +313,7 @@ You can alternatively provide an explicit list of named parameters using the ``" } .. [[[end]]] -.. _canned_queries_options: +.. _queries_options: Additional query options ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -389,7 +389,7 @@ This example demonstrates both ``fragment`` and ``hide_sql``: `See here `__ for a demo of this in action. -.. _canned_queries_writable: +.. _queries_writable: Writable queries ~~~~~~~~~~~~~~~~ @@ -524,7 +524,7 @@ You can pre-populate form fields when the page first loads using a query string, If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. -.. _canned_queries_magic_parameters: +.. _queries_magic_parameters: Magic parameters ~~~~~~~~~~~~~~~~ @@ -621,7 +621,7 @@ The form presented at ``/mydatabase/add_message`` will have just a field for ``m Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. -.. _canned_queries_json_api: +.. _queries_json_api: JSON API for writable queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/upgrade-1.0a20.md b/docs/upgrade-1.0a20.md index fbc3f4a8..b27d8bd5 100644 --- a/docs/upgrade-1.0a20.md +++ b/docs/upgrade-1.0a20.md @@ -244,7 +244,7 @@ except (KeyError, TypeError): New code: ```python try: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] except (KeyError, TypeError): @@ -253,7 +253,7 @@ except (KeyError, TypeError): ### Update render functions to async -If your plugin's render function needs to call `datasette.get_canned_query()` or other async Datasette methods, it must be declared as async: +If your plugin's render function needs to call `datasette.get_query()` or other async Datasette methods, it must be declared as async: Old code: ```python @@ -268,7 +268,7 @@ New code: async def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): # ... if query_name: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] ``` diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py deleted file mode 100644 index ae2c74e0..00000000 --- a/tests/test_canned_queries.py +++ /dev/null @@ -1,473 +0,0 @@ -from bs4 import BeautifulSoup as Soup -from asgiref.sync import async_to_sync -import json -import pytest -import re -from .fixtures import make_app_client - - -def update_query(client, name, **kwargs): - async_to_sync(client.ds.invoke_startup)() - async_to_sync(client.ds.update_query)("data", name, **kwargs) - - -@pytest.fixture -def canned_write_client(tmpdir): - template_dir = tmpdir / "canned_write_templates" - template_dir.mkdir() - (template_dir / "query-data-update_name.html").write_text( - """ - {% extends "query.html" %} - {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} - """, - "utf-8", - ) - with make_app_client( - extra_databases={"data.db": "create table names (name text)"}, - template_dir=str(template_dir), - config={ - "databases": { - "data": { - "queries": { - "canned_read": {"sql": "select * from names"}, - "add_name": { - "sql": "insert into names (name) values (:name)", - "write": True, - "on_success_redirect": "/data/add_name?success", - }, - "add_name_specify_id": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", - "write": True, - "on_error_redirect": "/data/add_name_specify_id?error", - }, - "add_name_specify_id_with_error_in_on_success_message_sql": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select this is bad SQL", - "write": True, - }, - "delete_name": { - "sql": "delete from names where rowid = :rowid", - "write": True, - "on_success_message": "Name deleted", - "allow": {"id": "root"}, - }, - "update_name": { - "sql": "update names set name = :name where rowid = :rowid", - "params": ["rowid", "name", "extra"], - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.fixture -def canned_write_immutable_client(): - with make_app_client( - is_immutable=True, - config={ - "databases": { - "fixtures": { - "queries": { - "add": { - "sql": "insert into sortable (text) values (:text)", - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.asyncio -async def test_canned_query_with_named_parameter(ds_client): - response = await ds_client.get( - "/fixtures/neighborhood_search.json?text=town&_shape=arrays" - ) - assert response.json()["rows"] == [ - ["Corktown", "Detroit", "MI"], - ["Downtown", "Los Angeles", "CA"], - ["Downtown", "Detroit", "MI"], - ["Greektown", "Detroit", "MI"], - ["Koreatown", "Los Angeles", "CA"], - ["Mexicantown", "Detroit", "MI"], - ] - - -def test_insert(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - csrftoken_from=True, - cookies={"foo": "bar"}, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Query executed, 1 row affected", 1]] - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name?success" - - -def test_insert_blocked_cross_site(canned_write_client): - # A cross-site POST (browser-originated) must be blocked - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - headers={"sec-fetch-site": "cross-site"}, - ) - assert 403 == response.status - - -def test_insert_no_cookies_no_csrf(canned_write_client): - response = canned_write_client.post("/data/add_name", {"name": "Hello"}) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - - -def test_custom_success_message(canned_write_client): - response = canned_write_client.post( - "/data/delete_name", - {"rowid": 1}, - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - csrftoken_from=True, - ) - assert 302 == response.status - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Name deleted", 1]] == messages - - -def test_insert_error(canned_write_client): - canned_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert 302 == response.status - assert "/data/add_name_specify_id?error" == response.headers["Location"] - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["UNIQUE constraint failed: names.rowid", 3]] == messages - # How about with a custom error message? - update_query(canned_write_client, "add_name_specify_id", on_error_message="ERROR") - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert [["ERROR", 3]] == canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - - -def test_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 5, "name": "Should be OK"}, - csrftoken_from=True, - ) - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name_specify_id" - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Name added: Should be OK with rowid 5", 1]] - - -def test_error_in_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id_with_error_in_on_success_message_sql", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [ - ["Error running on_success_message_sql: no such column: bad", 3] - ] - - -def test_custom_params(canned_write_client): - response = canned_write_client.get("/data/update_name?extra=foo") - assert ( - '' - in response.text - ) - - -def test_canned_query_pages_no_vary_header(canned_write_client): - # These pages no longer embed per-cookie CSRF tokens, so they must not - # set Vary: Cookie - they should be cacheable across users. - assert "vary" not in canned_write_client.get("/data").headers - assert "vary" not in canned_write_client.get("/data/update_name").headers - - -def test_json_post_body(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - body=json.dumps({"name": ["Hello", "there"]}), - ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] - - -@pytest.mark.parametrize( - "headers,body,querystring", - ( - (None, "name=NameGoesHere", "?_json=1"), - ({"Accept": "application/json"}, "name=NameGoesHere", None), - (None, "name=NameGoesHere&_json=1", None), - (None, '{"name": "NameGoesHere", "_json": 1}', None), - ), -) -def test_json_response(canned_write_client, headers, body, querystring): - response = canned_write_client.post( - "/data/add_name" + (querystring or ""), - body=body, - headers=headers, - ) - assert 200 == response.status - assert response.headers["content-type"] == "application/json; charset=utf-8" - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": "/data/add_name?success", - } - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "NameGoesHere"}] - - -def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth shows the five public queries - anon_response = canned_write_client.get("/data.json") - query_names = {q["name"] for q in anon_response.json["queries"]} - assert query_names == { - "add_name_specify_id_with_error_in_on_success_message_sql", - "update_name", - "add_name_specify_id", - "canned_read", - "add_name", - } - assert anon_response.json["queries_more"] is False - - # With auth the database page preview shows the first five queries - response = canned_write_client.get( - "/data.json", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - ] - assert response.json["queries_more"] is True - - # The full query list endpoint includes the remaining query - response = canned_write_client.get( - "/data/-/queries.json?_size=10", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - {"name": "update_name", "private": False}, - ] - - -def test_canned_query_permissions(canned_write_client): - assert 403 == canned_write_client.get("/data/delete_name").status - assert 200 == canned_write_client.get("/data/update_name").status - cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})} - assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status - assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status - - -@pytest.fixture(scope="session") -def magic_parameters_client(): - with make_app_client( - extra_databases={"data.db": "create table logs (line text)"}, - config={ - "databases": { - "data": { - "queries": { - "runme_post": {"sql": "", "write": True}, - "runme_get": {"sql": ""}, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.parametrize( - "magic_parameter,expected_re", - [ - ("_actor_id", "root"), - ("_header_host", "localhost"), - ("_header_not_a_thing", ""), - ("_cookie_foo", "bar"), - ("_now_epoch", r"^\d+$"), - ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), - ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), - ("_random_chars_1", r"^\w$"), - ("_random_chars_10", r"^\w{10}$"), - ], -) -def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): - update_query( - magic_parameters_client, - "runme_post", - sql=f"insert into logs (line) values (:{magic_parameter})", - ) - update_query( - magic_parameters_client, - "runme_get", - sql=f"select :{magic_parameter} as result", - ) - cookies = { - "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), - "foo": "bar", - } - # Test the GET version - get_response = magic_parameters_client.get( - "/data/runme_get.json?_shape=array", cookies=cookies - ) - get_actual = get_response.json[0]["result"] - assert re.match(expected_re, str(get_actual)) - # Test the form - form_response = magic_parameters_client.get("/data/runme_post") - soup = Soup(form_response.body, "html.parser") - # The magic parameter should not be represented as a form field - assert None is soup.find("input", {"name": magic_parameter}) - # Submit the form to create a log line - response = magic_parameters_client.post( - "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies - ) - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": None, - } - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert re.match(expected_re, post_actual) - - -@pytest.mark.parametrize("use_csrf", [True, False]) -@pytest.mark.parametrize("return_json", [True, False]) -def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): - update_query( - magic_parameters_client, - "runme_post", - sql="insert into logs (line) values (:_header_host)", - ) - qs = "" - if return_json: - qs = "?_json=1" - response = magic_parameters_client.post( - f"/data/runme_post{qs}", - {}, - csrftoken_from=use_csrf or None, - ) - if return_json: - assert response.status == 200 - assert response.json["ok"], response.json - else: - assert response.status == 302 - messages = magic_parameters_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Query executed, 1 row affected", 1]] == messages - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert post_actual == "localhost" - - -def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): - response = magic_parameters_client.get( - "/data/-/query.json?sql=select+:_header_host&_shape=array" - ) - assert 400 == response.status - assert response.json["error"].startswith("You did not supply a value for binding") - - -def test_canned_write_custom_template(canned_write_client): - response = canned_write_client.get("/data/update_name") - assert response.status == 200 - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text - assert ( - "" - in response.text - ) - # And test for link rel=alternate while we're here: - assert ( - '' - in response.text - ) - assert ( - response.headers["link"] - == '; rel="alternate"; type="application/json+datasette"' - ) - - -def test_canned_write_query_disabled_for_immutable_database( - canned_write_immutable_client, -): - response = canned_write_immutable_client.get("/fixtures/add") - assert response.status == 200 - assert ( - "This query cannot be executed because the database is immutable." - in response.text - ) - assert '' in response.text - # Submitting form should get a forbidden error - response = canned_write_immutable_client.post( - "/fixtures/add", - {"text": "text"}, - csrftoken_from=True, - ) - assert response.status == 403 - assert "Database is immutable" in response.text diff --git a/tests/test_html.py b/tests/test_html.py index 8edb9f6e..a9de5e79 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -633,7 +633,7 @@ async def test_404_content_type(ds_client): @pytest.mark.asyncio -async def test_canned_query_default_title(ds_client): +async def test_stored_query_default_title(ds_client): response = await ds_client.get("/fixtures/magic_parameters") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -641,7 +641,7 @@ async def test_canned_query_default_title(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_custom_metadata(ds_client): +async def test_stored_query_with_custom_metadata(ds_client): response = await ds_client.get("/fixtures/neighborhood_search?text=town") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -700,7 +700,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): +async def test_stored_query_with_hide_has_no_hidden_sql(ds_client): # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") @@ -720,7 +720,7 @@ async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): (True, "?_show_sql=1", "_show_sql", "/_memory/one", "hide"), ), ) -def test_canned_query_show_hide_metadata_option( +def test_stored_query_show_hide_metadata_option( hide_sql, querystring, expected_hidden, @@ -981,10 +981,10 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ("/fixtures/magic_parameters", None), ], ) -async def test_edit_sql_link_on_canned_queries(ds_client, path, expected): +async def test_edit_sql_link_on_stored_queries(ds_client, path, expected): response = await ds_client.get(path) assert response.status_code == 200 - expected_link = f'Edit SQL' + expected_link = f'Edit SQL' if expected: assert expected_link in response.text else: diff --git a/tests/test_jump.py b/tests/test_jump.py index f60af0fd..513a809f 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -76,7 +76,7 @@ async def ds_for_jump(): @pytest.mark.asyncio -async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_jump): +async def test_jump_searches_tables_databases_views_and_stored_queries(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=content", actor={"id": "user"} ) @@ -98,7 +98,7 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju @pytest.mark.asyncio -async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): +async def test_jump_uses_stored_query_names_not_titles(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=datasette", actor={"id": "user"} ) From cafb6b9dbd77e438b81a7784c4434957b73fb856 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:20:29 -0700 Subject: [PATCH 1779/1866] Need is_trusted=True for the counters demo --- .github/workflows/deploy-latest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 166d33d0..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -76,6 +76,7 @@ jobs: "update counters set value = value + 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) await datasette.add_query( "counters", @@ -83,6 +84,7 @@ jobs: "update counters set value = value - 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) return inner EOF From e2864fc895603c1fdf03d3c55812a0a6e56779ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:21:09 -0700 Subject: [PATCH 1780/1866] test_stored_queries.py --- tests/test_stored_queries.py | 473 +++++++++++++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 tests/test_stored_queries.py diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py new file mode 100644 index 00000000..2c648d5f --- /dev/null +++ b/tests/test_stored_queries.py @@ -0,0 +1,473 @@ +from bs4 import BeautifulSoup as Soup +from asgiref.sync import async_to_sync +import json +import pytest +import re +from .fixtures import make_app_client + + +def update_query(client, name, **kwargs): + async_to_sync(client.ds.invoke_startup)() + async_to_sync(client.ds.update_query)("data", name, **kwargs) + + +@pytest.fixture +def stored_write_client(tmpdir): + template_dir = tmpdir / "stored_write_templates" + template_dir.mkdir() + (template_dir / "query-data-update_name.html").write_text( + """ + {% extends "query.html" %} + {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} + """, + "utf-8", + ) + with make_app_client( + extra_databases={"data.db": "create table names (name text)"}, + template_dir=str(template_dir), + config={ + "databases": { + "data": { + "queries": { + "stored_read": {"sql": "select * from names"}, + "add_name": { + "sql": "insert into names (name) values (:name)", + "write": True, + "on_success_redirect": "/data/add_name?success", + }, + "add_name_specify_id": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", + "write": True, + "on_error_redirect": "/data/add_name_specify_id?error", + }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, + "delete_name": { + "sql": "delete from names where rowid = :rowid", + "write": True, + "on_success_message": "Name deleted", + "allow": {"id": "root"}, + }, + "update_name": { + "sql": "update names set name = :name where rowid = :rowid", + "params": ["rowid", "name", "extra"], + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.fixture +def stored_write_immutable_client(): + with make_app_client( + is_immutable=True, + config={ + "databases": { + "fixtures": { + "queries": { + "add": { + "sql": "insert into sortable (text) values (:text)", + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.asyncio +async def test_stored_query_with_named_parameter(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?text=town&_shape=arrays" + ) + assert response.json()["rows"] == [ + ["Corktown", "Detroit", "MI"], + ["Downtown", "Los Angeles", "CA"], + ["Downtown", "Detroit", "MI"], + ["Greektown", "Detroit", "MI"], + ["Koreatown", "Los Angeles", "CA"], + ["Mexicantown", "Detroit", "MI"], + ] + + +def test_insert(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + csrftoken_from=True, + cookies={"foo": "bar"}, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" + + +def test_insert_blocked_cross_site(stored_write_client): + # A cross-site POST (browser-originated) must be blocked + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + headers={"sec-fetch-site": "cross-site"}, + ) + assert 403 == response.status + + +def test_insert_no_cookies_no_csrf(stored_write_client): + response = stored_write_client.post("/data/add_name", {"name": "Hello"}) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + + +def test_custom_success_message(stored_write_client): + response = stored_write_client.post( + "/data/delete_name", + {"rowid": 1}, + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + csrftoken_from=True, + ) + assert 302 == response.status + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Name deleted", 1]] == messages + + +def test_insert_error(stored_write_client): + stored_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert 302 == response.status + assert "/data/add_name_specify_id?error" == response.headers["Location"] + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["UNIQUE constraint failed: names.rowid", 3]] == messages + # How about with a custom error message? + update_query(stored_write_client, "add_name_specify_id", on_error_message="ERROR") + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert [["ERROR", 3]] == stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + + +def test_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + +def test_custom_params(stored_write_client): + response = stored_write_client.get("/data/update_name?extra=foo") + assert ( + '' + in response.text + ) + + +def test_stored_query_pages_no_vary_header(stored_write_client): + # These pages no longer embed per-cookie CSRF tokens, so they must not + # set Vary: Cookie - they should be cacheable across users. + assert "vary" not in stored_write_client.get("/data").headers + assert "vary" not in stored_write_client.get("/data/update_name").headers + + +def test_json_post_body(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + body=json.dumps({"name": ["Hello", "there"]}), + ) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] + + +@pytest.mark.parametrize( + "headers,body,querystring", + ( + (None, "name=NameGoesHere", "?_json=1"), + ({"Accept": "application/json"}, "name=NameGoesHere", None), + (None, "name=NameGoesHere&_json=1", None), + (None, '{"name": "NameGoesHere", "_json": 1}', None), + ), +) +def test_json_response(stored_write_client, headers, body, querystring): + response = stored_write_client.post( + "/data/add_name" + (querystring or ""), + body=body, + headers=headers, + ) + assert 200 == response.status + assert response.headers["content-type"] == "application/json; charset=utf-8" + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": "/data/add_name?success", + } + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "NameGoesHere"}] + + +def test_stored_query_permissions_on_database_page(stored_write_client): + # Without auth shows the five public queries + anon_response = stored_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "update_name", + "add_name_specify_id", + "stored_read", + "add_name", + } + assert anon_response.json["queries_more"] is False + + # With auth the database page preview shows the first five queries + response = stored_write_client.get( + "/data.json", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = stored_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + {"name": "update_name", "private": False}, + ] + + +def test_stored_query_permissions(stored_write_client): + assert 403 == stored_write_client.get("/data/delete_name").status + assert 200 == stored_write_client.get("/data/update_name").status + cookies = {"ds_actor": stored_write_client.actor_cookie({"id": "root"})} + assert 200 == stored_write_client.get("/data/delete_name", cookies=cookies).status + assert 200 == stored_write_client.get("/data/update_name", cookies=cookies).status + + +@pytest.fixture(scope="session") +def magic_parameters_client(): + with make_app_client( + extra_databases={"data.db": "create table logs (line text)"}, + config={ + "databases": { + "data": { + "queries": { + "runme_post": {"sql": "", "write": True}, + "runme_get": {"sql": ""}, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.parametrize( + "magic_parameter,expected_re", + [ + ("_actor_id", "root"), + ("_header_host", "localhost"), + ("_header_not_a_thing", ""), + ("_cookie_foo", "bar"), + ("_now_epoch", r"^\d+$"), + ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), + ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), + ("_random_chars_1", r"^\w$"), + ("_random_chars_10", r"^\w{10}$"), + ], +) +def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): + update_query( + magic_parameters_client, + "runme_post", + sql=f"insert into logs (line) values (:{magic_parameter})", + ) + update_query( + magic_parameters_client, + "runme_get", + sql=f"select :{magic_parameter} as result", + ) + cookies = { + "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), + "foo": "bar", + } + # Test the GET version + get_response = magic_parameters_client.get( + "/data/runme_get.json?_shape=array", cookies=cookies + ) + get_actual = get_response.json[0]["result"] + assert re.match(expected_re, str(get_actual)) + # Test the form + form_response = magic_parameters_client.get("/data/runme_post") + soup = Soup(form_response.body, "html.parser") + # The magic parameter should not be represented as a form field + assert None is soup.find("input", {"name": magic_parameter}) + # Submit the form to create a log line + response = magic_parameters_client.post( + "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies + ) + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": None, + } + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert re.match(expected_re, post_actual) + + +@pytest.mark.parametrize("use_csrf", [True, False]) +@pytest.mark.parametrize("return_json", [True, False]) +def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): + update_query( + magic_parameters_client, + "runme_post", + sql="insert into logs (line) values (:_header_host)", + ) + qs = "" + if return_json: + qs = "?_json=1" + response = magic_parameters_client.post( + f"/data/runme_post{qs}", + {}, + csrftoken_from=use_csrf or None, + ) + if return_json: + assert response.status == 200 + assert response.json["ok"], response.json + else: + assert response.status == 302 + messages = magic_parameters_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Query executed, 1 row affected", 1]] == messages + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert post_actual == "localhost" + + +def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): + response = magic_parameters_client.get( + "/data/-/query.json?sql=select+:_header_host&_shape=array" + ) + assert 400 == response.status + assert response.json["error"].startswith("You did not supply a value for binding") + + +def test_stored_write_custom_template(stored_write_client): + response = stored_write_client.get("/data/update_name") + assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text + assert ( + "" + in response.text + ) + # And test for link rel=alternate while we're here: + assert ( + '' + in response.text + ) + assert ( + response.headers["link"] + == '; rel="alternate"; type="application/json+datasette"' + ) + + +def test_stored_write_query_disabled_for_immutable_database( + stored_write_immutable_client, +): + response = stored_write_immutable_client.get("/fixtures/add") + assert response.status == 200 + assert ( + "This query cannot be executed because the database is immutable." + in response.text + ) + assert '' in response.text + # Submitting form should get a forbidden error + response = stored_write_immutable_client.post( + "/fixtures/add", + {"text": "text"}, + csrftoken_from=True, + ) + assert response.status == 403 + assert "Database is immutable" in response.text From ca4907ab6b01f5ba5fa241534be7c9557f269050 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:30:36 -0700 Subject: [PATCH 1781/1866] Make _save_queries_from_config a private method --- datasette/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 52d100cc..6b66a53b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -598,7 +598,7 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file - async def apply_queries_config(self): + async def _save_queries_from_config(self): # Apply configured query entries from datasette.yaml to the internal table. await self.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -788,7 +788,7 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() - await self.apply_queries_config() + await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): From e89ffa0e0634a306be37bdd4894057329fee54a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:37:21 -0700 Subject: [PATCH 1782/1866] Fixed broken test caused by apply_queries_config() rename --- tests/test_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0e38c876..e5e75432 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,7 +937,7 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() try: # Convert old-style resource to Resource object from datasette.resources import DatabaseResource, QueryResource, TableResource @@ -964,7 +964,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() @pytest.mark.asyncio From 1bcd99df90d09049616fae411c2b6cddb6a82c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:42:36 -0700 Subject: [PATCH 1783/1866] Refactor code from datasette.app into datasette.stored_queries The datasette/app.py file had grown a lot in this branch. --- datasette/app.py | 537 ++++++----------------------------- datasette/stored_queries.py | 544 ++++++++++++++++++++++++++++++++++++ 2 files changed, 634 insertions(+), 447 deletions(-) create mode 100644 datasette/stored_queries.py diff --git a/datasette/app.py b/datasette/app.py index 6b66a53b..dd54446a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType +from . import stored_queries from .views import Context from .views.database import ( database_download, @@ -108,7 +109,6 @@ from .utils import ( module_from_path, move_plugins_and_allow, move_table_config, - named_parameters, parse_metadata, resolve_env_secrets, resolve_routes, @@ -284,17 +284,6 @@ DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() -UNCHANGED = object() - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) @@ -599,34 +588,7 @@ class Datasette: # a warning to user that they should delete their metadata.json file async def _save_queries_from_config(self): - # Apply configured query entries from datasette.yaml to the internal table. - await self.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((self.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await self.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) + await stored_queries.save_queries_from_config(self) def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env @@ -1067,46 +1029,11 @@ class Datasette: @staticmethod def _query_row_to_dict(row): - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - is_write = bool(row["is_write"]) - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": is_write, - "write": is_write, - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return stored_queries.query_row_to_dict(row) @staticmethod def _query_options_json(options): - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) + return stored_queries.query_options_json(options) async def add_query( self, @@ -1132,57 +1059,28 @@ class Datasette: on_error_redirect=None, replace=True, ): - parameters_json = json.dumps(list(parameters or [])) - options_json = self._query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await self.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], + return await stored_queries.add_query( + self, + database, + name, + sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + replace=replace, ) async def update_query( @@ -1190,113 +1088,52 @@ class Datasette: database, name, *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, + sql=stored_queries.UNCHANGED, + title=stored_queries.UNCHANGED, + description=stored_queries.UNCHANGED, + description_html=stored_queries.UNCHANGED, + hide_sql=stored_queries.UNCHANGED, + fragment=stored_queries.UNCHANGED, + parameters=stored_queries.UNCHANGED, + is_write=stored_queries.UNCHANGED, + is_private=stored_queries.UNCHANGED, + is_trusted=stored_queries.UNCHANGED, + source=stored_queries.UNCHANGED, + owner_id=stored_queries.UNCHANGED, + on_success_message=stored_queries.UNCHANGED, + on_success_message_sql=stored_queries.UNCHANGED, + on_success_redirect=stored_queries.UNCHANGED, + on_error_message=stored_queries.UNCHANGED, + on_error_redirect=stored_queries.UNCHANGED, ): - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value - for field, value in option_fields.items() - if value is not UNCHANGED - } - if changed_options: - rows = await self.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await self.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, + return await stored_queries.update_query( + self, + database, + name, + sql=sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, ) async def remove_query(self, database, name, source=None): - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await self.get_internal_database().execute_write(sql, params) + return await stored_queries.remove_query(self, database, name, source=source) async def get_query(self, database, name): - rows = await self.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return self._query_row_to_dict(rows.first()) + return await stored_queries.get_query(self, database, name) async def count_queries( self, @@ -1310,62 +1147,17 @@ class Datasette: source=None, owner_id=None, ): - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.count_queries( + self, + database, actor=actor, - parent=database, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await self.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] async def list_queries( self, @@ -1382,176 +1174,27 @@ class Datasette: owner_id=None, include_private=False, ): - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.list_queries( + self, + database, actor=actor, - parent=database, - include_is_private=include_private, + limit=limit, + cursor=cursor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + include_private=include_private, ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await self.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = self._query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None ): - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - db = self.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: - continue - if access.database != database: - raise Forbidden("Writable queries may not write to attached databases") - if not await self.allowed( - action=action, - resource=TableResource(database=access.database, table=access.table), - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) - return analysis + return await stored_queries.ensure_query_write_permissions( + self, database, sql, actor=actor, params=params, analysis=analysis + ) # Column types API diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py new file mode 100644 index 00000000..e88902e7 --- /dev/null +++ b/datasette/stored_queries.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import json + +from .resources import TableResource +from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components +from .utils.asgi import Forbidden + + +UNCHANGED = object() + +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + + +async def save_queries_from_config(datasette): + # Apply configured query entries from datasette.yaml to the internal table. + await datasette.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await datasette.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + + +def query_row_to_dict(row): + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") + is_write = bool(row["is_write"]) + return { + "database": row["database_name"], + "name": row["name"], + "sql": row["sql"], + "title": row["title"], + "description": row["description"], + "description_html": row["description_html"], + "hide_sql": bool(options.get("hide_sql")), + "fragment": options.get("fragment"), + "params": parameters, + "parameters": parameters, + "is_write": is_write, + "write": is_write, + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), + "source": row["source"], + "owner_id": row["owner_id"], + "on_success_message": options.get("on_success_message"), + "on_success_message_sql": options.get("on_success_message_sql"), + "on_success_redirect": options.get("on_success_redirect"), + "on_error_message": options.get("on_error_message"), + "on_error_redirect": options.get("on_error_redirect"), + } + + +def query_options_json(options): + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + + +async def add_query( + datasette, + database, + name, + sql, + *, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + is_private=False, + is_trusted=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, +): + parameters_json = json.dumps(list(parameters or [])) + options_json = query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + options = excluded.options, + parameters = excluded.parameters, + is_write = excluded.is_write, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, + source = excluded.source, + owner_id = excluded.owner_id, + updated_at = CURRENT_TIMESTAMP + """ + await datasette.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + options_json, + parameters_json, + int(bool(is_write)), + int(bool(is_private)), + int(bool(is_trusted)), + source, + owner_id, + ], + ) + + +async def update_query( + datasette, + database, + name, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +): + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "parameters": parameters, + "is_write": is_write, + "is_private": is_private, + "is_trusted": is_trusted, + "source": source, + "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"is_write", "is_private", "is_trusted"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + changed_options = { + field: value for field, value in option_fields.items() if value is not UNCHANGED + } + if changed_options: + rows = await datasette.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await datasette.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + +async def remove_query(datasette, database, name, source=None): + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await datasette.get_internal_database().execute_write(sql, params) + + +async def get_query(datasette, database, name): + rows = await datasette.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return query_row_to_dict(rows.first()) + + +async def count_queries( + datasette, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, +): + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await datasette.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + + +async def list_queries( + datasette, + database=None, + *, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + include_private=False, +): + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, + ) + params = dict(allowed_params) + params.update({"limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await datasette.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY {order_by} + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = query_row_to_dict(row) + if include_private: + query["private"] = bool(row["private"]) + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return { + "queries": queries, + "next": next_token, + "has_more": has_more, + "limit": limit, + } + + +async def ensure_query_write_permissions( + datasette, database, sql, *, actor=None, params=None, analysis=None +): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for access in analysis.table_accesses: + action = write_actions.get(access.operation) + if action is None: + continue + if access.database != database: + raise Forbidden("Writable queries may not write to attached databases") + if not await datasette.allowed( + action=action, + resource=TableResource(database=access.database, table=access.table), + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {action} on {access.database}/{access.table}" + ) + return analysis From 58e2e3a8ab9adfeda2686be5213eaf3f4118da33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:43:34 -0700 Subject: [PATCH 1784/1866] Ran cog --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index cec47f97..f8b1dc49 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json From c3ceabae03c2c9cf78f63f7c12c8d90301097cb4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:51:40 -0700 Subject: [PATCH 1785/1866] Ran Black --- datasette/stored_queries.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index e88902e7..5b005c07 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -6,7 +6,6 @@ from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components from .utils.asgi import Forbidden - UNCHANGED = object() QUERY_OPTION_FIELDS = ( From d6de8e752083a014bdd7ffb701dc08519da82554 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:52:16 -0700 Subject: [PATCH 1786/1866] Link to save query from /-/execute-write --- datasette/templates/execute_write.html | 29 +++++++++++++++++++++++++- datasette/views/database.py | 22 +++++++++++++++++++ tests/test_queries.py | 17 +++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 7a627a7a..6b626f8d 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -119,7 +119,10 @@ {% endif %}
    -

    +

    + + {% if save_query_base_url %}Save this query{% endif %} +

    ", + "on_success_message_sql": "select 'secret'", + } + }, + ) + form_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + data={ + "name": "unsafe_form", + "sql": "select 1", + "description_html": "", + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + assert form_response.status_code == 400 + assert "Invalid keys: description_html" in form_response.text + assert await ds.get_query("data", "unsafe") is None + assert await ds.get_query("data", "unsafe_form") is None + + @pytest.mark.asyncio async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) @@ -959,6 +1000,42 @@ async def test_query_update_and_delete_api(): assert await ds.get_query("data", "editable") is None +@pytest.mark.asyncio +async def test_query_update_api_rejects_config_only_fields(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_update_config_only_fields", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "description_html": "", + "on_success_message_sql": "select 'secret'", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + query = await ds.get_query("data", "editable") + assert query["description_html"] is None + assert query["on_success_message_sql"] is None + + @pytest.mark.asyncio async def test_query_update_api_rejects_trusted_queries_but_internal_update_allowed(): ds = Datasette( From b1289a73f9869e83a433a088c2a6c48285e67f2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 16:51:00 -0700 Subject: [PATCH 1795/1866] stored_queries.StoredQuery dataclass --- datasette/app.py | 102 ++++++------ datasette/stored_queries.py | 258 ++++++++++++++++++++---------- datasette/views/database.py | 56 +++---- datasette/views/query_helpers.py | 19 +-- datasette/views/stored_queries.py | 37 +++-- docs/internals.rst | 14 +- tests/test_queries.py | 128 +++++++-------- 7 files changed, 357 insertions(+), 257 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 96683895..56b89789 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1029,8 +1029,8 @@ class Datasette: ) @staticmethod - def _query_row_to_dict(row): - return stored_queries.query_row_to_dict(row) + def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: + return stored_queries.query_row_to_stored_query(row) @staticmethod def _query_options_json(options): @@ -1038,28 +1038,28 @@ class Datasette: async def add_query( self, - database, - name, - sql, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, - ): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, + ) -> None: return await stored_queries.add_query( self, database, @@ -1086,8 +1086,8 @@ class Datasette: async def update_query( self, - database, - name, + database: str, + name: str, *, sql=stored_queries.UNCHANGED, title=stored_queries.UNCHANGED, @@ -1106,7 +1106,7 @@ class Datasette: on_success_redirect=stored_queries.UNCHANGED, on_error_message=stored_queries.UNCHANGED, on_error_redirect=stored_queries.UNCHANGED, - ): + ) -> None: return await stored_queries.update_query( self, database, @@ -1130,24 +1130,28 @@ class Datasette: on_error_redirect=on_error_redirect, ) - async def remove_query(self, database, name, source=None): + async def remove_query( + self, database: str, name: str, source: str | None = None + ) -> None: return await stored_queries.remove_query(self, database, name, source=source) - async def get_query(self, database, name): + async def get_query( + self, database: str, name: str + ) -> stored_queries.StoredQuery | None: return await stored_queries.get_query(self, database, name) async def count_queries( self, - database=None, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - ): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + ) -> int: return await stored_queries.count_queries( self, database, @@ -1162,19 +1166,19 @@ class Datasette: async def list_queries( self, - database=None, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, - ): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, + ) -> stored_queries.StoredQueryPage: return await stored_queries.list_queries( self, database, diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index a28b71bf..bcfdfdb4 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,6 +1,8 @@ from __future__ import annotations +from dataclasses import dataclass import json +from typing import Any, Iterable from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components @@ -19,7 +21,76 @@ QUERY_OPTION_FIELDS = ( ) -async def save_queries_from_config(datasette): +@dataclass +class StoredQuery: + database: str + name: str + sql: str + title: str | None + description: str | None + description_html: str | None + hide_sql: bool + fragment: str | None + parameters: list[str] + is_write: bool + is_private: bool + is_trusted: bool + source: str + owner_id: str | None + on_success_message: str | None + on_success_message_sql: str | None + on_success_redirect: str | None + on_error_message: str | None + on_error_redirect: str | None + private: bool | None = None + + +@dataclass +class StoredQueryPage: + queries: list[StoredQuery] + next: str | None + has_more: bool + limit: int + + +def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: + data = { + "database": query.database, + "name": query.name, + "sql": query.sql, + "title": query.title, + "description": query.description, + "description_html": query.description_html, + "hide_sql": query.hide_sql, + "fragment": query.fragment, + "params": list(query.parameters), + "parameters": list(query.parameters), + "is_write": query.is_write, + "is_private": query.is_private, + "is_trusted": query.is_trusted, + "source": query.source, + "owner_id": query.owner_id, + "on_success_message": query.on_success_message, + "on_success_message_sql": query.on_success_message_sql, + "on_success_redirect": query.on_success_redirect, + "on_error_message": query.on_error_message, + "on_error_redirect": query.on_error_redirect, + } + if query.private is not None: + data["private"] = query.private + return data + + +def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: + return { + "queries": [stored_query_to_dict(query) for query in page.queries], + "next": page.next, + "has_more": page.has_more, + "limit": page.limit, + } + + +async def save_queries_from_config(datasette: Any) -> None: # Apply configured query entries from datasette.yaml to the internal table. await datasette.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -50,36 +121,38 @@ async def save_queries_from_config(datasette): ) -def query_row_to_dict(row): +def query_row_to_stored_query( + row: Any, private: bool | None = None +) -> StoredQuery | None: if row is None: return None parameters = json.loads(row["parameters"] or "[]") options = json.loads(row["options"] or "{}") - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": bool(row["is_write"]), - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return StoredQuery( + database=row["database_name"], + name=row["name"], + sql=row["sql"], + title=row["title"], + description=row["description"], + description_html=row["description_html"], + hide_sql=bool(options.get("hide_sql")), + fragment=options.get("fragment"), + parameters=parameters, + is_write=bool(row["is_write"]), + is_private=bool(row["is_private"]), + is_trusted=bool(row["is_trusted"]), + source=row["source"], + owner_id=row["owner_id"], + on_success_message=options.get("on_success_message"), + on_success_message_sql=options.get("on_success_message_sql"), + on_success_redirect=options.get("on_success_redirect"), + on_error_message=options.get("on_error_message"), + on_error_redirect=options.get("on_error_redirect"), + private=private, + ) -def query_options_json(options): +def query_options_json(options: dict[str, Any]) -> str: options_dict = {} for field in QUERY_OPTION_FIELDS: value = options.get(field) @@ -92,29 +165,29 @@ def query_options_json(options): async def add_query( - datasette, - database, - name, - sql, + datasette: Any, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, +) -> None: parameters_json = json.dumps(list(parameters or [])) options_json = query_options_json( { @@ -170,9 +243,9 @@ async def add_query( async def update_query( - datasette, - database, - name, + datasette: Any, + database: str, + name: str, *, sql=UNCHANGED, title=UNCHANGED, @@ -191,7 +264,7 @@ async def update_query( on_success_redirect=UNCHANGED, on_error_message=UNCHANGED, on_error_redirect=UNCHANGED, -): +) -> None: fields = { "sql": sql, "title": title, @@ -263,7 +336,9 @@ async def update_query( ) -async def remove_query(datasette, database, name, source=None): +async def remove_query( + datasette: Any, database: str, name: str, source: str | None = None +) -> None: sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" params = [database, name] if source is not None: @@ -272,7 +347,7 @@ async def remove_query(datasette, database, name, source=None): await datasette.get_internal_database().execute_write(sql, params) -async def get_query(datasette, database, name): +async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: rows = await datasette.get_internal_database().execute( """ SELECT * FROM queries @@ -280,21 +355,21 @@ async def get_query(datasette, database, name): """, [database, name], ) - return query_row_to_dict(rows.first()) + return query_row_to_stored_query(rows.first()) async def count_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, +) -> int: allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", actor=actor, @@ -354,20 +429,20 @@ async def count_queries( async def list_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, -): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, +) -> StoredQueryPage: limit = min(max(1, int(limit)), 1000) allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", @@ -480,9 +555,10 @@ async def list_queries( queries = [] for row in rows: - query = query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) + query = query_row_to_stored_query( + row, private=bool(row["private"]) if include_private else None + ) + assert query is not None queries.append(query) next_token = None @@ -499,17 +575,23 @@ async def list_queries( tilde_encode(last_row["sort_key"]), tilde_encode(last_row["name"]), ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } + return StoredQueryPage( + queries=queries, + next=next_token, + has_more=has_more, + limit=limit, + ) async def ensure_query_write_permissions( - datasette, database, sql, *, actor=None, params=None, analysis=None -): + datasette: Any, + database: str, + sql: str, + *, + actor: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + analysis: Any = None, +) -> Any: write_actions = { "insert": "insert-row", "update": "update-row", diff --git a/datasette/views/database.py b/datasette/views/database.py index 98ca989c..b558b002 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,6 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -99,8 +100,8 @@ class DatabaseView(View): limit=5, include_private=True, ) - stored_queries = queries_page["queries"] - queries_more = queries_page["has_more"] + stored_queries = queries_page.queries + queries_more = queries_page.has_more queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more @@ -136,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": stored_queries, + "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -447,7 +448,7 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=stored_query["name"]), + resource=QueryResource(database=db.name, query=stored_query.name), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") @@ -480,20 +481,18 @@ class QueryView(View): or request.args.get("_json") or params.get("_json") ) - params_for_query = MagicParameters( - stored_query["sql"], params, request, datasette - ) + params_for_query = MagicParameters(stored_query.sql, params, request, datasette) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - stored_query["sql"], params_for_query, request=request + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = stored_query.get("on_success_message_sql") + on_success_message_sql = stored_query.on_success_message_sql if on_success_message_sql: try: message_result = ( @@ -505,18 +504,19 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = stored_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" + message = ( + stored_query.on_success_message + or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) ) - redirect_url = stored_query.get("on_success_redirect") + redirect_url = stored_query.on_success_redirect ok = True except Exception as ex: - message = stored_query.get("on_error_message") or str(ex) + message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR - redirect_url = stored_query.get("on_error_redirect") + redirect_url = stored_query.on_error_redirect ok = False if should_return_json: return Response.json( @@ -562,7 +562,7 @@ class QueryView(View): ) if stored_query is None: raise - stored_query_write = bool(stored_query.get("is_write")) + stored_query_write = stored_query.is_write private = False if stored_query: @@ -570,7 +570,7 @@ class QueryView(View): visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=stored_query["name"]), + resource=QueryResource(database=database, query=stored_query.name), ) if not visible: raise Forbidden("You do not have permission to view this query") @@ -591,14 +591,14 @@ class QueryView(View): sql = None if stored_query: - sql = stored_query["sql"] + sql = stored_query.sql elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if stored_query and stored_query.get("params"): - named_parameters = stored_query["params"] + if stored_query and stored_query.parameters: + named_parameters = stored_query.parameters if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -686,7 +686,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, database=database, table=None, request=request, @@ -721,7 +721,7 @@ class QueryView(View): if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query.name)}.html", ) environment = datasette.get_jinja_environment(request) @@ -740,7 +740,7 @@ class QueryView(View): ) metadata = await datasette.get_database_metadata(database) if stored_query: - metadata = dict(stored_query) + metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) renderers = {} @@ -775,7 +775,7 @@ class QueryView(View): ) show_hide_hidden = "" - if stored_query and stored_query.get("hide_sql"): + if stored_query and stored_query.hide_sql: if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -843,7 +843,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, request=request, sql=sql, params=params, @@ -863,7 +863,7 @@ class QueryView(View): "sql": sql, "params": params, }, - stored_query=stored_query["name"] if stored_query else None, + stored_query=stored_query.name if stored_query else None, private=private, stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, @@ -907,7 +907,7 @@ class QueryView(View): datasette, request, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index de732431..46d71b8e 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -2,6 +2,7 @@ import json import re from datasette.resources import DatabaseResource, TableResource +from datasette.stored_queries import StoredQuery from datasette.utils import ( named_parameters as derive_named_parameters, escape_sqlite, @@ -281,18 +282,18 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis -async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): - if query.get("is_trusted"): +async def _ensure_stored_query_execution_permissions( + datasette, db, query: StoredQuery, actor +): + if query.is_trusted: return - if query.get("is_write"): + if query.is_write: await datasette.ensure_permission( action="execute-write-sql", resource=DatabaseResource(db.name), actor=actor, ) - await datasette.ensure_query_write_permissions( - db.name, query["sql"], actor=actor - ) + await datasette.ensure_query_write_permissions(db.name, query.sql, actor=actor) else: await datasette.ensure_permission( action="execute-sql", @@ -482,7 +483,7 @@ async def _prepare_query_create(datasette, request, db, data): } -async def _prepare_query_update(datasette, request, db, existing, update): +async def _prepare_query_update(datasette, request, db, existing: StoredQuery, update): invalid_keys = set(update) - _query_update_fields if invalid_keys: raise QueryValidationError( @@ -490,8 +491,8 @@ async def _prepare_query_update(datasette, request, db, existing, update): ) update = _apply_query_data_types(update) - sql = update.get("sql", existing["sql"]) - query_is_write = existing["is_write"] + sql = update.get("sql", existing.sql) + query_is_write = existing.is_write derived = _derived_query_parameters(sql) parameters = None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 1a2c5d00..8c4e849e 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -1,6 +1,7 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import sqlite3, tilde_decode from datasette.utils.asgi import Response @@ -100,7 +101,7 @@ class QueryListView(BaseView): ) query_list_path = self.query_list_path(database) next_url = None - if page["next"]: + if page.next: pairs = [ (key, value) for key, value in parse_qsl( @@ -108,7 +109,7 @@ class QueryListView(BaseView): ) if key != "_next" ] - pairs.append(("_next", page["next"])) + pairs.append(("_next", page.next)) next_url = "{}?{}".format( query_list_path, urlencode(pairs), @@ -194,13 +195,13 @@ class QueryListView(BaseView): "database_color": ( self.ds.get_database(database).color if database is not None else None ), - "queries": page["queries"], - "next": page["next"], + "queries": page.queries, + "next": page.next, "next_url": next_url, - "has_more": page["has_more"], - "limit": page["limit"], - "show_private_note": any(query["is_private"] for query in page["queries"]), - "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), + "has_more": page.has_more, + "limit": page.limit, + "show_private_note": any(query.is_private for query in page.queries), + "show_trusted_note": any(query.is_trusted for query in page.queries), "query_list_path": query_list_path, "show_database": database is None, "facets": facets, @@ -213,7 +214,12 @@ class QueryListView(BaseView): }, } if format_ == "json": - return Response.json(data) + return Response.json( + { + **data, + "queries": [stored_query_to_dict(query) for query in page.queries], + } + ) return await self.render( ["query_list.html"], request, @@ -374,8 +380,11 @@ class QueryStoreView(QueryCreateView): return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) + assert query is not None if is_json: - return Response.json({"ok": True, "query": query}, status=201) + return Response.json( + {"ok": True, "query": stored_query_to_dict(query)}, status=201 + ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -395,7 +404,7 @@ class QueryDefinitionView(BaseView): actor=request.actor, ): return _error(["Permission denied"], 403) - return Response.json({"ok": True, "query": query}) + return Response.json({"ok": True, "query": stored_query_to_dict(query)}) class QueryUpdateView(BaseView): @@ -413,7 +422,7 @@ class QueryUpdateView(BaseView): actor=request.actor, ): return _error(["Permission denied: need update-query"], 403) - if existing.get("is_trusted"): + if existing.is_trusted: return _error(["Trusted queries cannot be updated using the API"], 403) try: @@ -444,10 +453,12 @@ class QueryUpdateView(BaseView): await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): + query = await self.ds.get_query(db.name, query_name) + assert query is not None return Response.json( { "ok": True, - "query": await self.ds.get_query(db.name, query_name), + "query": stored_query_to_dict(query), } ) return Response.json({"ok": True}) diff --git a/docs/internals.rst b/docs/internals.rst index 66724aa9..4980ee8b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1039,11 +1039,11 @@ Example: await .get_query(database, name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns a stored query dictionary, or ``None`` if the query does not exist. +Returns a ``StoredQuery`` dataclass instance, or ``None`` if the query does not exist. -The dictionary contains ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``params``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. +``StoredQuery`` has the following attributes: ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. -``parameters`` and ``params`` contain the same list of explicit parameter names. +``parameters`` is a list of explicit parameter names. .. _datasette_list_queries: @@ -1087,12 +1087,12 @@ Lists stored queries visible to the specified actor. ``owner_id`` - string, optional Filter by owner actor ID. ``include_private`` - boolean, optional - Set to ``True`` to include a ``private`` boolean in each returned query dictionary indicating if anonymous users would be unable to view that query. + Set to ``True`` to populate a ``private`` boolean on each returned ``StoredQuery`` indicating if anonymous users would be unable to view that query. -The return value is a dictionary with these keys: +The return value is a ``StoredQueryPage`` dataclass instance with these attributes: -``queries`` - list of dictionaries - Stored query dictionaries, in the same format returned by :ref:`datasette_get_query`. +``queries`` - list of StoredQuery instances + Stored queries in the same format returned by :ref:`datasette_get_query`. ``next`` - string or None Pagination cursor for the next page, if one exists. ``has_more`` - boolean diff --git a/tests/test_queries.py b/tests/test_queries.py index 70fb7a03..59fab8c0 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -4,6 +4,7 @@ import pytest from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden @@ -87,38 +88,41 @@ async def test_add_get_and_remove_query(): } query = await ds.get_query("data", "top_customers") - assert query == { - "database": "data", - "name": "top_customers", - "sql": "select * from customers where region = :region", - "title": "Top customers", - "description": "Customers by region", - "description_html": None, - "hide_sql": True, - "fragment": "chart", - "params": ["region"], - "parameters": ["region"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "user", - "owner_id": "alice", - "on_success_message": None, - "on_success_message_sql": None, - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert query == StoredQuery( + database="data", + name="top_customers", + sql="select * from customers where region = :region", + title="Top customers", + description="Customers by region", + description_html=None, + hide_sql=True, + fragment="chart", + parameters=["region"], + is_write=False, + is_private=False, + is_trusted=True, + source="user", + owner_id="alice", + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [query] - assert queries_page["next"] is None + assert queries_page == StoredQueryPage( + queries=[query], + next=None, + has_more=False, + limit=50, + ) await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [] - assert queries_page["next"] is None + assert queries_page.queries == [] + assert queries_page.next is None @pytest.mark.asyncio @@ -156,13 +160,12 @@ async def test_update_query_only_updates_provided_fields(): ) query = await ds.get_query("data", "redirect") - assert query["title"] == "Updated" - assert query["parameters"] == [] - assert query["params"] == [] - assert query["on_success_redirect"] is None - assert query["sql"] == "select 1" - assert query["is_private"] is False - assert query["is_trusted"] is False + assert query.title == "Updated" + assert query.parameters == [] + assert query.on_success_redirect is None + assert query.sql == "select 1" + assert query.is_private is False + assert query.is_trusted is False options_row = ( await ds.get_internal_database().execute( """ @@ -198,28 +201,27 @@ async def test_config_queries_imported_to_internal_table(): ds.add_memory_database("query_config", name="data") await ds.invoke_startup() - assert await ds.get_query("data", "configured") == { - "database": "data", - "name": "configured", - "sql": "select :name as name", - "title": "Configured query", - "description": None, - "description_html": "

    Configured HTML

    ", - "hide_sql": False, - "fragment": None, - "params": ["name"], - "parameters": ["name"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "config", - "owner_id": None, - "on_success_message": None, - "on_success_message_sql": "select 'Hello ' || :name", - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert await ds.get_query("data", "configured") == StoredQuery( + database="data", + name="configured", + sql="select :name as name", + title="Configured query", + description=None, + description_html="

    Configured HTML

    ", + hide_sql=False, + fragment=None, + parameters=["name"], + is_write=False, + is_private=False, + is_trusted=True, + source="config", + owner_id=None, + on_success_message=None, + on_success_message_sql="select 'Hello ' || :name", + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) @pytest.mark.asyncio @@ -1032,8 +1034,8 @@ async def test_query_update_api_rejects_config_only_fields(): "Invalid keys: description_html, on_success_message_sql" ] query = await ds.get_query("data", "editable") - assert query["description_html"] is None - assert query["on_success_message_sql"] is None + assert query.description_html is None + assert query.on_success_message_sql is None @pytest.mark.asyncio @@ -1072,9 +1074,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo "Trusted queries cannot be updated using the API" ] query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 1 as one" - assert query["title"] == "Original" + assert query.is_trusted is True + assert query.sql == "select 1 as one" + assert query.title == "Original" await ds.update_query( "data", @@ -1083,9 +1085,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo title="Internal", ) query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 3 as three" - assert query["title"] == "Internal" + assert query.is_trusted is True + assert query.sql == "select 3 as three" + assert query.title == "Internal" @pytest.mark.asyncio From 9f66cf72c1c9170f10e863d750ac4eee47113a7f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 21:42:50 -0700 Subject: [PATCH 1796/1866] Removed execute write SQL from query create page --- datasette/templates/query_create.html | 7 ------- 1 file changed, 7 deletions(-) diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index f5dadbff..ec910456 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -106,9 +106,6 @@ form.sql .query-create-sql textarea#sql-editor { .query-create-analysis-note { margin: 0; } -.query-create-action { - margin: 0.35rem 0 1rem; -} .query-create-analysis { margin-top: 0.8rem; } @@ -171,10 +168,6 @@ form.sql .query-create-sql textarea#sql-editor { Queries marked private can only be seen by you, their creator.

    - {% if sql and analysis_is_write %} -

    Execute write SQL

    - {% endif %} -

    From 737ff03efbb2bdc99b10d2654b7818526ec51e13 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 22:11:06 -0700 Subject: [PATCH 1797/1866] Expanded analysis of SQL operations, refs #2748 --- datasette/permissions.py | 10 ++ datasette/stored_queries.py | 137 +++++++++++++-- datasette/utils/sql_analysis.py | 289 +++++++++++++++++++++++++++---- datasette/views/execute_write.py | 9 +- datasette/views/query_helpers.py | 104 +++++++---- tests/test_actions_sql.py | 14 +- tests/test_internals_database.py | 34 ++-- tests/test_queries.py | 166 ++++++++++++++++++ tests/test_utils_sql_analysis.py | 97 +++++++++-- 9 files changed, 740 insertions(+), 120 deletions(-) diff --git a/datasette/permissions.py b/datasette/permissions.py index 917c58ab..a9a3cc7c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -58,6 +58,16 @@ class Resource(ABC): self.child = child self._private = None # Sentinel to track if private was set + def __str__(self) -> str: + return "/".join( + str(part) for part in (self.parent, self.child) if part is not None + ) + + def __repr__(self) -> str: + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child + ) + @property def private(self) -> bool: """ diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index bcfdfdb4..c4b083e5 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -2,11 +2,16 @@ from __future__ import annotations from dataclasses import dataclass import json -from typing import Any, Iterable +from typing import Any, Iterable, TYPE_CHECKING -from .resources import TableResource +from .resources import DatabaseResource, TableResource +from .permissions import Resource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components from .utils.asgi import Forbidden +from .utils.sql_analysis import Operation, SQLAnalysis + +if TYPE_CHECKING: + from .app import Datasette UNCHANGED = object() @@ -583,20 +588,94 @@ async def list_queries( ) -async def ensure_query_write_permissions( - datasette: Any, - database: str, - sql: str, - *, - actor: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - analysis: Any = None, -) -> Any: +PermissionRequirement = tuple[str, Resource] + + +def permission_for_operation(operation: Operation) -> PermissionRequirement | None: write_actions = { "insert": "insert-row", "update": "update-row", "delete": "delete-row", } + action = write_actions.get(operation.operation) + if ( + action + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + action, + TableResource(database=operation.database, table=operation.table), + ) + if operation.operation == "create" and operation.target_type == "table": + if operation.database is None: + return None + return ( + "create-table", + DatabaseResource(database=operation.database), + ) + if ( + operation.operation == "alter" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "alter-table", + TableResource(database=operation.database, table=operation.table), + ) + if ( + operation.operation == "drop" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "drop-table", + TableResource(database=operation.database, table=operation.table), + ) + if ( + operation.operation in {"create", "drop"} + and operation.target_type == "index" + and operation.database is not None + and operation.table is not None + ): + return ( + "alter-table", + TableResource(database=operation.database, table=operation.table), + ) + return None + + +def operation_is_write(operation: Operation) -> bool: + return operation.operation in { + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "attach", + "detach", + "pragma", + "analyze", + "reindex", + } + + +async def ensure_query_write_permissions( + datasette: Datasette, + database: str, + sql: str, + *, + actor: dict[str, object] | None = None, + params: dict[str, object] | None = None, + analysis: SQLAnalysis | None = None, +) -> SQLAnalysis: db = datasette.get_database(database) if analysis is None: if params is None: @@ -606,18 +685,38 @@ async def ensure_query_write_permissions( except sqlite3.DatabaseError as ex: raise Forbidden(f"Could not analyze query: {ex}") from ex - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: + has_semantic_schema_operation = any( + operation.operation in {"create", "alter", "drop"} + and operation.target_type in {"table", "index", "view", "trigger"} + for operation in analysis.operations + ) + for operation in analysis.operations: + if operation.internal and has_semantic_schema_operation: continue - if access.database != database: + if has_semantic_schema_operation and operation.operation in { + "read", + "insert", + "update", + "delete", + "reindex", + }: + continue + permission = permission_for_operation(operation) + if permission is None: + if operation_is_write(operation): + raise Forbidden( + "Unsupported SQL operation: {} {}".format( + operation.operation, operation.target_type + ) + ) + continue + action, resource = permission + if operation.database != database: raise Forbidden("Writable queries may not write to attached databases") if not await datasette.allowed( action=action, - resource=TableResource(database=access.database, table=access.table), + resource=resource, actor=actor, ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) + raise Forbidden(f"Permission denied: need {action} on {resource}") return analysis diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index b5317b62..54f310fe 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -3,22 +3,66 @@ from typing import Literal from datasette.utils.sqlite import sqlite3 +SQLOperation = Literal[ + "read", + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "attach", + "detach", + "pragma", + "analyze", + "reindex", +] +SQLTargetType = Literal[ + "table", + "index", + "view", + "trigger", + "schema", + "transaction", + "database", + "pragma", + "unknown", +] SQLTableOperation = Literal["read", "insert", "update", "delete"] @dataclass(frozen=True) -class SQLTableAccess: - operation: SQLTableOperation +class Operation: + operation: SQLOperation + target_type: SQLTargetType database: str | None - table: str + table: str | None sqlite_schema: str | None + target: str | None = None columns: tuple[str, ...] = () source: str | None = None + internal: bool = False @dataclass(frozen=True) class SQLAnalysis: - table_accesses: tuple[SQLTableAccess, ...] + operations: tuple[Operation, ...] + + +# Hashable dict key for grouping repeated authorizer callbacks while collecting columns. +@dataclass(frozen=True) +class OperationKey: + operation: SQLOperation + target_type: SQLTargetType + database: str | None + table: str | None + sqlite_schema: str | None + target: str | None + source: str | None + internal: bool _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { @@ -28,6 +72,36 @@ _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { sqlite3.SQLITE_DELETE: "delete", } +# Values are (operation, target_type) pairs used to construct Operation objects. +_CREATE_ACTIONS = { + sqlite3.SQLITE_CREATE_INDEX: ("create", "index"), + sqlite3.SQLITE_CREATE_TABLE: ("create", "table"), + sqlite3.SQLITE_CREATE_TRIGGER: ("create", "trigger"), + sqlite3.SQLITE_CREATE_VIEW: ("create", "view"), +} +_DROP_ACTIONS = { + sqlite3.SQLITE_DROP_INDEX: ("drop", "index"), + sqlite3.SQLITE_DROP_TABLE: ("drop", "table"), + sqlite3.SQLITE_DROP_TRIGGER: ("drop", "trigger"), + sqlite3.SQLITE_DROP_VIEW: ("drop", "view"), +} +for action_name, operation, target_type in ( + ("SQLITE_CREATE_TEMP_INDEX", "create", "index"), + ("SQLITE_CREATE_TEMP_TABLE", "create", "table"), + ("SQLITE_CREATE_TEMP_TRIGGER", "create", "trigger"), + ("SQLITE_CREATE_TEMP_VIEW", "create", "view"), + ("SQLITE_DROP_TEMP_INDEX", "drop", "index"), + ("SQLITE_DROP_TEMP_TABLE", "drop", "table"), + ("SQLITE_DROP_TEMP_TRIGGER", "drop", "trigger"), + ("SQLITE_DROP_TEMP_VIEW", "drop", "view"), +): + action_value = getattr(sqlite3, action_name, None) + if action_value is not None: + actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS + actions[action_value] = (operation, target_type) + +_SQLITE_SCHEMA_TABLES = {"sqlite_master", "sqlite_schema"} + def analyze_sql_tables( conn, @@ -38,15 +112,13 @@ def analyze_sql_tables( schema_to_database: dict[str, str] | None = None, ) -> SQLAnalysis: """ - Return tables accessed by a SQL statement according to SQLite's authorizer. + Return operations performed by a SQL statement according to SQLite's authorizer. This function is synchronous and connection-based. It temporarily installs a - SQLite authorizer, prepares ``EXPLAIN ``, and returns the table access + SQLite authorizer, prepares ``EXPLAIN ``, and returns the operation callbacks observed while SQLite compiles the statement. """ - accesses: dict[ - tuple[SQLTableOperation, str | None, str, str | None, str | None], set[str] - ] = {} + operations: dict[OperationKey, set[str]] = {} def database_for_schema(sqlite_schema): if schema_to_database and sqlite_schema in schema_to_database: @@ -55,21 +127,166 @@ def analyze_sql_tables( return database_name return sqlite_schema + def record( + operation: SQLOperation, + target_type: SQLTargetType, + *, + database: str | None, + table: str | None, + sqlite_schema: str | None, + target: str | None, + source: str | None, + column: str | None = None, + internal: bool = False, + ): + key = OperationKey( + operation=operation, + target_type=target_type, + database=database, + table=table, + sqlite_schema=sqlite_schema, + target=target, + source=source, + internal=internal, + ) + columns = operations.setdefault(key, set()) + if column is not None: + columns.add(column) + def authorizer(action, arg1, arg2, sqlite_schema, source): operation = _ACTION_TO_OPERATION.get(action) - if operation is None or arg1 is None: + if operation is not None and arg1 is not None: + target_type = "schema" if arg1 in _SQLITE_SCHEMA_TABLES else "table" + column = ( + arg2 if operation in ("read", "update") and arg2 is not None else None + ) + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=arg1 if target_type == "table" else None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + column=column, + internal=target_type == "schema", + ) + return sqlite3.SQLITE_OK + + create_operation = _CREATE_ACTIONS.get(action) + if create_operation is not None and arg1 is not None: + operation, target_type = create_operation + related_table = arg2 if target_type in {"index", "trigger"} else arg1 + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=related_table, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + drop_operation = _DROP_ACTIONS.get(action) + if drop_operation is not None and arg1 is not None: + operation, target_type = drop_operation + related_table = arg2 if target_type in {"index", "trigger"} else arg1 + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=related_table, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ALTER_TABLE and arg2 is not None: + record( + "alter", + "table", + database=database_for_schema(arg1), + table=arg2, + sqlite_schema=arg1, + target=arg2, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_TRANSACTION and arg1 is not None: + record( + arg1.lower(), + "transaction", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ATTACH and arg1 is not None: + record( + "attach", + "database", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_DETACH and arg1 is not None: + record( + "detach", + "database", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_PRAGMA and arg1 is not None: + record( + "pragma", + "pragma", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ANALYZE: + record( + "analyze", + "database" if arg1 is None else "table", + database=database_for_schema(sqlite_schema), + table=arg1, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_REINDEX and arg1 is not None: + record( + "reindex", + "index", + database=database_for_schema(sqlite_schema), + table=None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) return sqlite3.SQLITE_OK - key = ( - operation, - database_for_schema(sqlite_schema), - arg1, - sqlite_schema, - source, - ) - columns = accesses.setdefault(key, set()) - if operation in ("read", "update") and arg2 is not None: - columns.add(arg2) return sqlite3.SQLITE_OK conn.set_authorizer(authorizer) @@ -78,22 +295,26 @@ def analyze_sql_tables( finally: conn.set_authorizer(None) + has_schema_operation = any( + key.target_type in {"table", "index", "view", "trigger"} + and key.operation in {"create", "alter", "drop"} + for key in operations + ) + return SQLAnalysis( - table_accesses=tuple( - SQLTableAccess( - operation=operation, - database=database, - table=table, - sqlite_schema=sqlite_schema, + operations=tuple( + Operation( + operation=key.operation, + target_type=key.target_type, + database=key.database, + table=key.table, + sqlite_schema=key.sqlite_schema, + target=key.target, columns=tuple(sorted(columns)), - source=source, + source=key.source, + internal=key.internal + or (has_schema_operation and key.target_type == "schema"), ) - for ( - operation, - database, - table, - sqlite_schema, - source, - ), columns in accesses.items() + for key, columns in operations.items() ) ) diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 0054300c..cead8926 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -193,9 +193,12 @@ class ExecuteWriteView(BaseView): status=400, ) - message = "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) + if cursor.rowcount == -1: + message = "Query executed" + else: + message = "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) if _wants_json(request, is_json, data): return _block_framing( Response.json( diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 46d71b8e..922f4e52 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -1,8 +1,12 @@ import json import re -from datasette.resources import DatabaseResource, TableResource -from datasette.stored_queries import StoredQuery +from datasette.resources import DatabaseResource +from datasette.stored_queries import ( + StoredQuery, + operation_is_write, + permission_for_operation, +) from datasette.utils import ( named_parameters as derive_named_parameters, escape_sqlite, @@ -12,6 +16,7 @@ from datasette.utils import ( InvalidSql, ) from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -123,11 +128,8 @@ def _coerce_query_parameters(value, derived): return parameters -def _analysis_is_write(analysis): - return any( - access.operation in {"insert", "update", "delete"} - for access in analysis.table_accesses - ) +def _analysis_is_write(analysis: SQLAnalysis) -> bool: + return any(operation_is_write(operation) for operation in analysis.operations) def _block_framing(response): @@ -201,34 +203,66 @@ async def _analyze_user_query(datasette, db, sql, *, actor): return is_write, derived, analysis -def _analysis_rows(analysis): - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - return [ - { - "operation": access.operation, - "database": access.database, - "table": access.table, - "required_permission": write_actions.get(access.operation, ""), - "source": access.source, - } - for access in analysis.table_accesses - ] +def _semantic_schema_operation_is_present(operations: tuple[Operation, ...]) -> bool: + return any( + operation.operation in {"create", "alter", "drop"} + and operation.target_type in {"table", "index", "view", "trigger"} + for operation in operations + ) -async def _analysis_rows_with_permissions(datasette, analysis, actor): +def _display_operations(analysis: SQLAnalysis) -> list[Operation]: + has_semantic_schema_operation = _semantic_schema_operation_is_present( + analysis.operations + ) + operations = [] + for operation in analysis.operations: + if operation.internal and has_semantic_schema_operation: + continue + if has_semantic_schema_operation and operation.operation in { + "read", + "insert", + "update", + "delete", + "reindex", + }: + continue + operations.append(operation) + return operations + + +def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: + rows = [] + for operation in _display_operations(analysis): + permission = permission_for_operation(operation) + required_permission = permission[0] if permission else "" + rows.append( + { + "operation": operation.operation, + "database": operation.database, + "table": operation.table or operation.target, + "required_permission": required_permission, + "source": operation.source, + } + ) + return rows + + +async def _analysis_rows_with_permissions( + datasette, analysis: SQLAnalysis, actor +) -> list[dict[str, object]]: rows = _analysis_rows(analysis) - for row in rows: - permission = row["required_permission"] + for row, operation in zip(rows, _display_operations(analysis)): + permission = permission_for_operation(operation) if permission: + action, resource = permission row["allowed"] = await datasette.allowed( - action=permission, - resource=TableResource(row["database"], row["table"]), + action=action, + resource=resource, actor=actor, ) + elif operation_is_write(operation): + row["allowed"] = False else: row["allowed"] = None return rows @@ -398,15 +432,19 @@ async def _inserted_row_url(datasette, db, analysis, cursor): if lastrowid is None: return None direct_inserts = [ - access - for access in analysis.table_accesses - if access.operation == "insert" - and access.source is None - and access.database == db.name + operation + for operation in analysis.operations + if operation.operation == "insert" + and operation.target_type == "table" + and not operation.internal + and operation.source is None + and operation.database == db.name ] if len(direct_inserts) != 1: return None table = direct_inserts[0].table + if table is None: + return None pks = await db.primary_keys(table) use_rowid = not pks select = ( diff --git a/tests/test_actions_sql.py b/tests/test_actions_sql.py index 863d2529..a1fca971 100644 --- a/tests/test_actions_sql.py +++ b/tests/test_actions_sql.py @@ -12,10 +12,22 @@ import pytest import pytest_asyncio from datasette.app import Datasette from datasette.permissions import PermissionSQL -from datasette.resources import TableResource +from datasette.resources import DatabaseResource, QueryResource, TableResource from datasette import hookimpl +def test_resource_string_representations(): + assert str(DatabaseResource("content")) == "content" + assert repr(DatabaseResource("content")) == ( + "DatabaseResource(parent='content', child=None)" + ) + assert str(TableResource("content", "dogs")) == "content/dogs" + assert repr(TableResource("content", "dogs")) == ( + "TableResource(parent='content', child='dogs')" + ) + assert str(QueryResource("content", "insert-a-dog")) == "content/insert-a-dog" + + # Test plugin that provides permission rules class PermissionRulesPlugin: def __init__(self, rules_callback): diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 5481a398..d6e130b4 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -698,14 +698,17 @@ async def test_analyze_sql(): assert [ ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal ] == [ ("read", "data", "main", "dogs", ("id", "name"), None), ] @@ -722,14 +725,17 @@ async def test_analyze_sql_insert_select(): assert { ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal } == { ("insert", "data", "main", "dogs", (), None), ("read", "data", "main", "cats", ("name",), None), diff --git a/tests/test_queries.py b/tests/test_queries.py index 59fab8c0..4b8a6486 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1643,6 +1643,172 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_create_table_uses_create_table_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "insert-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["creator", "row-writer"]}, + "execute-write-sql": {"id": ["creator", "row-writer"]}, + "create-table": {"id": "creator"}, + } + } + }, + }, + ) + db = ds.add_memory_database("execute_write_create_table", name="data") + await ds.invoke_startup() + + analysis_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "creator"}, + params={"sql": "create table foobar (id integer primary key, name text)"}, + ) + allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create table foobar (id integer primary key, name text)"}, + ) + row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create table should_not_exist (id integer primary key)"}, + ) + + assert analysis_response.status_code == 200 + analysis_data = analysis_response.json() + assert analysis_data["ok"] is True + assert analysis_data["execute_disabled"] is False + assert analysis_data["analysis_rows"] == [ + { + "operation": "create", + "database": "data", + "table": "foobar", + "required_permission": "create-table", + "source": None, + "allowed": True, + } + ] + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert allowed_response.json()["message"] == "Query executed" + assert await db.table_exists("foobar") + + assert row_permission_response.status_code == 403 + assert row_permission_response.json()["errors"] == [ + "Permission denied: need create-table on data" + ] + assert not await db.table_exists("should_not_exist") + + +@pytest.mark.asyncio +async def test_execute_write_alter_and_drop_table_use_schema_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "delete-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["alterer", "dropper", "row-writer"]}, + "execute-write-sql": { + "id": ["alterer", "dropper", "row-writer"] + }, + }, + "tables": { + "dogs": { + "permissions": { + "alter-table": {"id": "alterer"}, + "drop-table": {"id": "dropper"}, + } + } + }, + } + }, + }, + ) + db = ds.add_memory_database("execute_write_alter_drop_table", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + await ds.invoke_startup() + + alter_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "alter table dogs add column age integer"}, + ) + alter_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "alter table cats add column age integer"}, + ) + + assert alter_allowed_response.status_code == 200 + assert "age" in [column.name for column in await db.table_column_details("dogs")] + assert alter_row_permission_response.status_code == 403 + assert alter_row_permission_response.json()["errors"] == [ + "Permission denied: need alter-table on data/cats" + ] + assert "age" not in [ + column.name for column in await db.table_column_details("cats") + ] + + create_index_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "create index idx_dogs_name on dogs(name)"}, + ) + create_index_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create index idx_cats_name on cats(name)"}, + ) + drop_index_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "drop index idx_dogs_name"}, + ) + + assert create_index_allowed_response.status_code == 200 + assert create_index_row_permission_response.status_code == 403 + assert create_index_row_permission_response.json()["errors"] == [ + "Permission denied: need alter-table on data/cats" + ] + assert drop_index_allowed_response.status_code == 200 + + drop_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "dropper"}, + json={"sql": "drop table dogs"}, + ) + drop_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "drop table cats"}, + ) + + assert drop_allowed_response.status_code == 200 + assert not await db.table_exists("dogs") + assert drop_row_permission_response.status_code == 403 + assert drop_row_permission_response.json()["errors"] == [ + "Permission denied: need drop-table on data/cats" + ] + assert await db.table_exists("cats") + + @pytest.mark.asyncio async def test_execute_write_insert_links_to_inserted_row(): ds = Datasette(memory=True, default_deny=True) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 5730cd0d..5306a515 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -26,17 +26,20 @@ def conn(): conn.close() -def as_tuples(analysis): +def table_operation_tuples(analysis): return [ ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal ] @@ -48,7 +51,7 @@ def test_analyze_select_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("read", "data", "main", "cats", ("id", "name"), None), ("read", "data", "main", "dogs", ("age", "id", "name"), None), } @@ -57,11 +60,73 @@ def test_analyze_select_tables(conn): def test_analyze_uses_sqlite_schema_as_default_database(conn): analysis = analyze_sql_tables(conn, "select name from dogs") - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("read", "main", "main", "dogs", ("name",), None), } +def operation_dict(operation): + return { + "operation": operation.operation, + "target_type": operation.target_type, + "database": operation.database, + "sqlite_schema": operation.sqlite_schema, + "table": operation.table, + "target": operation.target, + "columns": operation.columns, + "source": operation.source, + "internal": operation.internal, + } + + +def test_analyze_create_table_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables( + conn, + "create table foobar (id integer primary key, name text)", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "create", + "target_type": "table", + "database": "data", + "sqlite_schema": "main", + "table": "foobar", + "target": "foobar", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + assert not [ + operation + for operation in analysis.operations + if operation.table in {"sqlite_master", "sqlite_schema"} + and not operation.internal + ] + + +def test_analyze_transaction_operation(conn): + analysis = analyze_sql_tables(conn, "commit", database_name="data") + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "commit", + "target_type": "transaction", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "COMMIT", + "columns": (), + "source": None, + "internal": False, + } + ] + + def test_analyze_insert_tables(conn): analysis = analyze_sql_tables( conn, @@ -70,7 +135,7 @@ def test_analyze_insert_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "data", "main", "dogs", (), None), ("read", "data", "main", "dogs", ("id", "name"), "dogs_after_insert"), ("update", "data", "main", "cats", ("name",), "dogs_after_insert"), @@ -87,7 +152,7 @@ def test_analyze_update_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("update", "data", "main", "dogs", ("age",), None), ("read", "data", "main", "dogs", ("age", "name"), None), } @@ -101,7 +166,7 @@ def test_analyze_delete_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("delete", "data", "main", "dogs", (), None), ("read", "data", "main", "dogs", ("name",), None), } @@ -121,7 +186,7 @@ def test_analyze_insert_select_with_cte(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "data", "main", "cats", (), None), ("read", "data", "main", "dogs", ("age", "name"), "old_dogs"), } @@ -135,7 +200,7 @@ def test_analyze_view_with_instead_of_trigger(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("update", "data", "main", "dog_names", ("name",), None), ("read", "data", "main", "dogs", ("id", "name"), "dog_names"), ("read", "data", "main", "dog_names", ("id", "name"), "dog_names"), @@ -163,7 +228,7 @@ def test_analyze_attached_database_tables(conn): schema_to_database={"extra": "extra_db"}, ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "extra_db", "extra", "people", (), None), ("read", "data", "main", "dogs", ("name",), None), } From 86d0e7335f98a88874df31ec0adb64967446dfac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 14:52:52 -0700 Subject: [PATCH 1798/1866] Deny unsupported write SQL operations by default Require view-table permission for reads discovered inside write SQL analysis, including INSERT ... SELECT and CREATE TABLE ... AS SELECT. Record additional SQLite authorizer callbacks as Operation values so unsupported functions, savepoints, virtual table DDL, and unknown callbacks are denied unless explicitly handled. --- datasette/stored_queries.py | 43 +++---- datasette/utils/sql_analysis.py | 192 +++++++++++++++++++++++++++++-- datasette/views/execute_write.py | 4 +- datasette/views/query_helpers.py | 32 ++---- tests/test_queries.py | 136 ++++++++++++++++++++-- tests/test_utils_sql_analysis.py | 94 +++++++++++++++ 6 files changed, 433 insertions(+), 68 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index c4b083e5..4b0fe6a6 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -592,6 +592,16 @@ PermissionRequirement = tuple[str, Resource] def permission_for_operation(operation: Operation) -> PermissionRequirement | None: + if ( + operation.operation == "read" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "view-table", + TableResource(database=operation.database, table=operation.table), + ) write_actions = { "insert": "insert-row", "update": "update-row", @@ -648,6 +658,10 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No return None +def operation_should_be_ignored(operation: Operation) -> bool: + return operation.internal or operation.operation == "select" + + def operation_is_write(operation: Operation) -> bool: return operation.operation in { "insert", @@ -659,11 +673,13 @@ def operation_is_write(operation: Operation) -> bool: "begin", "commit", "rollback", + "savepoint", "attach", "detach", "pragma", "analyze", "reindex", + "unknown", } @@ -685,34 +701,19 @@ async def ensure_query_write_permissions( except sqlite3.DatabaseError as ex: raise Forbidden(f"Could not analyze query: {ex}") from ex - has_semantic_schema_operation = any( - operation.operation in {"create", "alter", "drop"} - and operation.target_type in {"table", "index", "view", "trigger"} - for operation in analysis.operations - ) for operation in analysis.operations: - if operation.internal and has_semantic_schema_operation: - continue - if has_semantic_schema_operation and operation.operation in { - "read", - "insert", - "update", - "delete", - "reindex", - }: + if operation_should_be_ignored(operation): continue permission = permission_for_operation(operation) if permission is None: - if operation_is_write(operation): - raise Forbidden( - "Unsupported SQL operation: {} {}".format( - operation.operation, operation.target_type - ) + raise Forbidden( + "Unsupported SQL operation: {} {}".format( + operation.operation, operation.target_type ) - continue + ) action, resource = permission if operation.database != database: - raise Forbidden("Writable queries may not write to attached databases") + raise Forbidden("Writable queries may not access attached databases") if not await datasette.allowed( action=action, resource=resource, diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 54f310fe..8963da77 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -8,30 +8,39 @@ SQLOperation = Literal[ "insert", "update", "delete", + "select", + "function", "create", "alter", "drop", "begin", "commit", "rollback", + "savepoint", "attach", "detach", "pragma", "analyze", "reindex", + "unknown", ] SQLTargetType = Literal[ "table", "index", "view", "trigger", + "virtual-table", "schema", + "statement", "transaction", "database", "pragma", + "function", "unknown", ] SQLTableOperation = Literal["read", "insert", "update", "delete"] +SQLSchemaOperation = Literal["create", "drop"] +SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] @dataclass(frozen=True) @@ -73,19 +82,34 @@ _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { } # Values are (operation, target_type) pairs used to construct Operation objects. -_CREATE_ACTIONS = { +_CREATE_ACTIONS: dict[int, tuple[SQLSchemaOperation, SQLSchemaTargetType]] = { sqlite3.SQLITE_CREATE_INDEX: ("create", "index"), sqlite3.SQLITE_CREATE_TABLE: ("create", "table"), sqlite3.SQLITE_CREATE_TRIGGER: ("create", "trigger"), sqlite3.SQLITE_CREATE_VIEW: ("create", "view"), } -_DROP_ACTIONS = { +_DROP_ACTIONS: dict[int, tuple[SQLSchemaOperation, SQLSchemaTargetType]] = { sqlite3.SQLITE_DROP_INDEX: ("drop", "index"), sqlite3.SQLITE_DROP_TABLE: ("drop", "table"), sqlite3.SQLITE_DROP_TRIGGER: ("drop", "trigger"), sqlite3.SQLITE_DROP_VIEW: ("drop", "view"), } -for action_name, operation, target_type in ( + + +def _add_schema_action( + action_name: str, + operation: SQLSchemaOperation, + target_type: SQLSchemaTargetType, +) -> None: + action_value = getattr(sqlite3, action_name, None) + if action_value is not None: + actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS + actions[action_value] = (operation, target_type) + + +_TEMP_SCHEMA_ACTIONS: tuple[ + tuple[str, SQLSchemaOperation, SQLSchemaTargetType], ... +] = ( ("SQLITE_CREATE_TEMP_INDEX", "create", "index"), ("SQLITE_CREATE_TEMP_TABLE", "create", "table"), ("SQLITE_CREATE_TEMP_TRIGGER", "create", "trigger"), @@ -94,13 +118,76 @@ for action_name, operation, target_type in ( ("SQLITE_DROP_TEMP_TABLE", "drop", "table"), ("SQLITE_DROP_TEMP_TRIGGER", "drop", "trigger"), ("SQLITE_DROP_TEMP_VIEW", "drop", "view"), -): - action_value = getattr(sqlite3, action_name, None) - if action_value is not None: - actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS - actions[action_value] = (operation, target_type) +) +for schema_action in _TEMP_SCHEMA_ACTIONS: + _add_schema_action(*schema_action) -_SQLITE_SCHEMA_TABLES = {"sqlite_master", "sqlite_schema"} +_VTABLE_SCHEMA_ACTIONS: tuple[ + tuple[str, SQLSchemaOperation, SQLSchemaTargetType], ... +] = ( + ("SQLITE_CREATE_VTABLE", "create", "virtual-table"), + ("SQLITE_DROP_VTABLE", "drop", "virtual-table"), +) +for schema_action in _VTABLE_SCHEMA_ACTIONS: + _add_schema_action(*schema_action) + +_SQLITE_SCHEMA_TABLES = { + "sqlite_master", + "sqlite_schema", + "sqlite_temp_master", + "sqlite_temp_schema", +} +_SQLITE_INTERNAL_SCHEMA_FUNCTIONS = { + "length", + "like", + "printf", + "sqlite_drop_column", + "sqlite_rename_column", + "sqlite_rename_quotefix", + "sqlite_rename_table", + "sqlite_rename_test", + "substr", +} + +_AUTHORIZER_ACTION_NAMES = { + getattr(sqlite3, name): name + for name in ( + "SQLITE_CREATE_INDEX", + "SQLITE_CREATE_TABLE", + "SQLITE_CREATE_TEMP_INDEX", + "SQLITE_CREATE_TEMP_TABLE", + "SQLITE_CREATE_TEMP_TRIGGER", + "SQLITE_CREATE_TEMP_VIEW", + "SQLITE_CREATE_TRIGGER", + "SQLITE_CREATE_VIEW", + "SQLITE_DELETE", + "SQLITE_DROP_INDEX", + "SQLITE_DROP_TABLE", + "SQLITE_DROP_TEMP_INDEX", + "SQLITE_DROP_TEMP_TABLE", + "SQLITE_DROP_TEMP_TRIGGER", + "SQLITE_DROP_TEMP_VIEW", + "SQLITE_DROP_TRIGGER", + "SQLITE_DROP_VIEW", + "SQLITE_INSERT", + "SQLITE_PRAGMA", + "SQLITE_READ", + "SQLITE_SELECT", + "SQLITE_TRANSACTION", + "SQLITE_UPDATE", + "SQLITE_ATTACH", + "SQLITE_DETACH", + "SQLITE_ALTER_TABLE", + "SQLITE_REINDEX", + "SQLITE_ANALYZE", + "SQLITE_CREATE_VTABLE", + "SQLITE_DROP_VTABLE", + "SQLITE_FUNCTION", + "SQLITE_SAVEPOINT", + "SQLITE_RECURSIVE", + ) + if hasattr(sqlite3, name) +} def analyze_sql_tables( @@ -287,6 +374,52 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_SELECT: + record( + "select", + "statement", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=None, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_FUNCTION and arg2 is not None: + record( + "function", + "function", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=arg2, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_SAVEPOINT and arg1 is not None: + record( + "savepoint", + "transaction", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) + record( + "unknown", + "unknown", + database=database_for_schema(sqlite_schema), + table=None, + sqlite_schema=sqlite_schema, + target=action_name, + source=source, + ) return sqlite3.SQLITE_OK conn.set_authorizer(authorizer) @@ -296,10 +429,46 @@ def analyze_sql_tables( conn.set_authorizer(None) has_schema_operation = any( - key.target_type in {"table", "index", "view", "trigger"} + key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} for key in operations ) + dropped_tables = { + (key.database, key.table) + for key in operations + if key.operation == "drop" and key.target_type == "table" + } + + def key_is_drop_table_delete(key: OperationKey) -> bool: + return ( + key.operation == "delete" + and key.target_type == "table" + and (key.database, key.table) in dropped_tables + ) + + has_user_table_access_in_schema_operation = any( + key.operation in {"read", "insert", "update", "delete"} + and key.target_type == "table" + and not key.internal + and not key_is_drop_table_delete(key) + for key in operations + ) + + def operation_is_internal(key: OperationKey) -> bool: + if key.internal or (has_schema_operation and key.target_type == "schema"): + return True + if has_schema_operation and key.operation == "reindex": + return True + if ( + has_schema_operation + and not has_user_table_access_in_schema_operation + and key.operation == "function" + and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS + ): + return True + if key_is_drop_table_delete(key): + return True + return False return SQLAnalysis( operations=tuple( @@ -312,8 +481,7 @@ def analyze_sql_tables( target=key.target, columns=tuple(sorted(columns)), source=key.source, - internal=key.internal - or (has_schema_operation and key.target_type == "schema"), + internal=operation_is_internal(key), ) for key, columns in operations.items() ) diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index cead8926..19006ac5 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -99,9 +99,7 @@ class ExecuteWriteView(BaseView): "parameter_names": parameter_names, "parameter_values": parameter_values, "analysis_error": analysis_error, - "analysis_rows": [ - row for row in analysis_rows if row["operation"] != "read" - ], + "analysis_rows": analysis_rows, "execution_message": execution_message, "execution_links": execution_links, "execution_ok": execution_ok, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 922f4e52..05a0d73e 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,6 +5,7 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, operation_is_write, + operation_should_be_ignored, permission_for_operation, ) from datasette.utils import ( @@ -203,29 +204,10 @@ async def _analyze_user_query(datasette, db, sql, *, actor): return is_write, derived, analysis -def _semantic_schema_operation_is_present(operations: tuple[Operation, ...]) -> bool: - return any( - operation.operation in {"create", "alter", "drop"} - and operation.target_type in {"table", "index", "view", "trigger"} - for operation in operations - ) - - def _display_operations(analysis: SQLAnalysis) -> list[Operation]: - has_semantic_schema_operation = _semantic_schema_operation_is_present( - analysis.operations - ) operations = [] for operation in analysis.operations: - if operation.internal and has_semantic_schema_operation: - continue - if has_semantic_schema_operation and operation.operation in { - "read", - "insert", - "update", - "delete", - "reindex", - }: + if operation_should_be_ignored(operation): continue operations.append(operation) return operations @@ -252,6 +234,7 @@ async def _analysis_rows_with_permissions( datasette, analysis: SQLAnalysis, actor ) -> list[dict[str, object]]: rows = _analysis_rows(analysis) + is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): permission = permission_for_operation(operation) if permission: @@ -261,7 +244,7 @@ async def _analysis_rows_with_permissions( resource=resource, actor=actor, ) - elif operation_is_write(operation): + elif is_write: row["allowed"] = False else: row["allowed"] = None @@ -360,7 +343,7 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): "ok": analysis_error is None, "parameters": parameter_names, "analysis_error": analysis_error, - "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "analysis_rows": analysis_rows, "execute_disabled": bool( (not sql) or analysis_error @@ -374,6 +357,7 @@ async def _query_create_analysis_data(datasette, db, sql, actor): parameter_names = [] analysis_rows = [] analysis_error = None + analysis: SQLAnalysis | None = None if has_sql: try: parameter_names = _derived_query_parameters(sql) @@ -390,9 +374,7 @@ async def _query_create_analysis_data(datasette, db, sql, actor): "analysis_error": analysis_error, "analysis_rows": analysis_rows, "has_sql": has_sql, - "analysis_is_write": bool( - analysis_rows and any(row["required_permission"] for row in analysis_rows) - ), + "analysis_is_write": _analysis_is_write(analysis) if analysis else False, "save_disabled": bool( (not has_sql) or analysis_error diff --git a/tests/test_queries.py b/tests/test_queries.py index 4b8a6486..97ec973f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1181,11 +1181,10 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert 'Required permission' in create_response.text assert 'Source' not in create_response.text assert "read" in create_response.text + assert "view-table" in create_response.text assert ( - create_response.text.count( - 'n/a' - ) - == 2 + 'n/a' + not in create_response.text ) assert create_response.text.index( 'value="Save query"' @@ -1255,9 +1254,9 @@ async def test_create_query_analyze_endpoint_uses_sql_only(): "operation": "read", "database": "data", "table": "dogs", - "required_permission": "", + "required_permission": "view-table", "source": None, - "allowed": None, + "allowed": True, } ] @@ -1375,7 +1374,8 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'Required permission' in response.text assert "insert" in response.text assert "update" in response.text - assert "read" not in response.text + assert "read" in response.text + assert "view-table" in response.text assert 'action="/data/-/execute-write"' in response.text assert "insert into dogs (name) values ('Cleo')" in response.text assert (await db.execute("select count(*) from dogs")).first()[0] == 0 @@ -1643,6 +1643,127 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_select_requires_view_table_on_source(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + }, + "public_log": {"permissions": {"insert-row": {"id": "writer"}}}, + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_insert_select_source", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("create table public_log (value text)") + await db.execute_write("insert into secret values ('sensitive')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "insert into public_log(value) select value from secret"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need view-table on data/secret" + ] + assert (await db.execute("select value from public_log")).dicts() == [] + + +@pytest.mark.asyncio +async def test_execute_write_create_table_as_select_requires_view_table_on_source(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "creator"}, + "execute-write-sql": {"id": "creator"}, + "create-table": {"id": "creator"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_create_as_select_source", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("insert into secret values ('sensitive')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create table copied_secret as select value from secret"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need view-table on data/secret" + ] + assert not await db.table_exists("copied_secret") + + +@pytest.mark.asyncio +async def test_execute_write_rejects_function_operations(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_function_operation", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "insert into dogs (name) values (upper('cleo'))"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: function function" + ] + assert (await db.execute("select name from dogs")).dicts() == [] + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( @@ -1733,6 +1854,7 @@ async def test_execute_write_alter_and_drop_table_use_schema_permissions(): "permissions": { "alter-table": {"id": "alterer"}, "drop-table": {"id": "dropper"}, + "view-table": {"id": "alterer"}, } } }, diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 5306a515..2ae11502 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -127,6 +127,100 @@ def test_analyze_transaction_operation(conn): ] +def test_analyze_savepoint_operation(conn): + analysis = analyze_sql_tables(conn, "savepoint s", database_name="data") + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "savepoint", + "target_type": "transaction", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "BEGIN s", + "columns": (), + "source": None, + "internal": False, + } + ] + + +def test_analyze_function_operation(conn): + analysis = analyze_sql_tables( + conn, + "insert into dogs (name) values (upper(:name))", + {"name": "Cleo"}, + database_name="data", + ) + + assert { + ( + operation.operation, + operation.target_type, + operation.target, + operation.database, + operation.table, + ) + for operation in analysis.operations + } == { + ("insert", "table", "dogs", "data", "dogs"), + ("function", "function", "upper", None, None), + ("read", "table", "dogs", "data", "dogs"), + ("update", "table", "cats", "data", "cats"), + ("read", "table", "cats", "data", "cats"), + ("insert", "table", "log", "data", "log"), + } + + +def test_analyze_create_virtual_table_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables( + conn, + "create virtual table docs using fts5(body)", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "create", + "target_type": "virtual-table", + "database": "data", + "sqlite_schema": "main", + "table": "docs", + "target": "docs", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + +def test_analyze_create_table_as_select_function_is_not_internal(): + conn = sqlite3.connect(":memory:") + try: + conn.execute("create table secret(value text)") + analysis = analyze_sql_tables( + conn, + "create table copied as select substr(value, 1, 1) from secret", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "function", + "target_type": "function", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "substr", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + def test_analyze_insert_tables(conn): analysis = analyze_sql_tables( conn, From 03b2c66f6312b8317d87eb4c1326977f6f63b26d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 15:17:10 -0700 Subject: [PATCH 1799/1866] Require full row mutation permissions for raw SQL Raw SQL insert and update statements can have broader effects than their SQLite authorizer callbacks reveal. INSERT OR REPLACE and UPDATE OR REPLACE can delete conflicting rows while only surfacing insert or update operations. Expand table insert and update operations to require insert-row, update-row, and delete-row together. Keep delete operations mapped to delete-row, and update the analysis UI/API to report and evaluate multiple required permissions for a single operation. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559083539 --- datasette/stored_queries.py | 108 ++++++++++++----- datasette/views/query_helpers.py | 27 +++-- tests/test_queries.py | 200 ++++++++++++++++++++++++++++++- 3 files changed, 290 insertions(+), 45 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index 4b0fe6a6..cf44a9ff 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -588,10 +588,25 @@ async def list_queries( ) -PermissionRequirement = tuple[str, Resource] +@dataclass(frozen=True) +class PermissionRequirement: + action: str + resource: Resource -def permission_for_operation(operation: Operation) -> PermissionRequirement | None: +def row_mutation_requirements( + database: str, table: str +) -> tuple[PermissionRequirement, ...]: + resource = TableResource(database=database, table=table) + return tuple( + PermissionRequirement(action=action, resource=resource) + for action in ("insert-row", "update-row", "delete-row") + ) + + +def permission_requirements_for_operation( + operation: Operation, +) -> tuple[PermissionRequirement, ...]: if ( operation.operation == "read" and operation.target_type == "table" @@ -599,31 +614,45 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "view-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="view-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - action = write_actions.get(operation.operation) if ( - action + operation.operation in {"insert", "update"} + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return row_mutation_requirements( + database=operation.database, + table=operation.table, + ) + if ( + operation.operation == "delete" and operation.target_type == "table" and operation.database is not None and operation.table is not None ): return ( - action, - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="delete-row", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if operation.operation == "create" and operation.target_type == "table": if operation.database is None: - return None + return () return ( - "create-table", - DatabaseResource(database=operation.database), + PermissionRequirement( + action="create-table", + resource=DatabaseResource(database=operation.database), + ), ) if ( operation.operation == "alter" @@ -632,8 +661,12 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "alter-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if ( operation.operation == "drop" @@ -642,8 +675,12 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "drop-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="drop-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if ( operation.operation in {"create", "drop"} @@ -652,10 +689,14 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "alter-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) - return None + return () def operation_should_be_ignored(operation: Operation) -> bool: @@ -704,20 +745,23 @@ async def ensure_query_write_permissions( for operation in analysis.operations: if operation_should_be_ignored(operation): continue - permission = permission_for_operation(operation) - if permission is None: + permissions = permission_requirements_for_operation(operation) + if not permissions: raise Forbidden( "Unsupported SQL operation: {} {}".format( operation.operation, operation.target_type ) ) - action, resource = permission if operation.database != database: raise Forbidden("Writable queries may not access attached databases") - if not await datasette.allowed( - action=action, - resource=resource, - actor=actor, - ): - raise Forbidden(f"Permission denied: need {action} on {resource}") + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {permission.action} " + f"on {permission.resource}" + ) return analysis diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 05a0d73e..7f3ef1bc 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -6,7 +6,7 @@ from datasette.stored_queries import ( StoredQuery, operation_is_write, operation_should_be_ignored, - permission_for_operation, + permission_requirements_for_operation, ) from datasette.utils import ( named_parameters as derive_named_parameters, @@ -216,8 +216,10 @@ def _display_operations(analysis: SQLAnalysis) -> list[Operation]: def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): - permission = permission_for_operation(operation) - required_permission = permission[0] if permission else "" + permissions = permission_requirements_for_operation(operation) + required_permission = ", ".join( + permission.action for permission in permissions + ) rows.append( { "operation": operation.operation, @@ -236,14 +238,17 @@ async def _analysis_rows_with_permissions( rows = _analysis_rows(analysis) is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): - permission = permission_for_operation(operation) - if permission: - action, resource = permission - row["allowed"] = await datasette.allowed( - action=action, - resource=resource, - actor=actor, - ) + permissions = permission_requirements_for_operation(operation) + if permissions: + row["allowed"] = True + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + row["allowed"] = False + break elif is_write: row["allowed"] = False else: diff --git a/tests/test_queries.py b/tests/test_queries.py index 97ec973f..fcd19d1c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -508,6 +508,8 @@ async def test_analyze_write_query_requires_table_permissions(): "dogs": { "permissions": { "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, } } } @@ -1429,7 +1431,7 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): "operation": "insert", "database": "data", "table": "dogs", - "required_permission": "insert-row", + "required_permission": "insert-row, update-row, delete-row", "source": None, "allowed": True, } @@ -1627,6 +1629,40 @@ async def test_execute_write_post_requires_database_and_table_permissions(): } } } + missing_update_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert missing_update_permission.status_code == 403 + assert missing_update_permission.json()["errors"] == [ + "Permission denied: need update-row on data/dogs" + ] + + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ + "update-row" + ] = {"id": "writer"} + missing_delete_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert missing_delete_permission.status_code == 403 + assert missing_delete_permission.json()["errors"] == [ + "Permission denied: need delete-row on data/dogs" + ] + + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ + "delete-row" + ] = {"id": "writer"} allowed = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1643,6 +1679,156 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_or_replace_requires_delete_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_insert_or_replace", name="data") + await db.execute_write( + "create table users (id integer primary key, email text unique)" + ) + await db.execute_write( + "insert into users (id, email) values " + "(1, 'a@example.com'), (2, 'b@example.com')" + ) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": ( + "insert or replace into users(id, email) " + "values (3, 'b@example.com')" + ) + }, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need delete-row on data/users" + ] + assert (await db.execute("select id, email from users order by id")).dicts() == [ + {"id": 1, "email": "a@example.com"}, + {"id": 2, "email": "b@example.com"}, + ] + + +@pytest.mark.asyncio +async def test_execute_write_update_or_replace_requires_delete_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_update_or_replace", name="data") + await db.execute_write( + "create table users (id integer primary key, email text unique)" + ) + await db.execute_write( + "insert into users (id, email) values " + "(1, 'a@example.com'), (2, 'b@example.com')" + ) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "update or replace users set email = 'b@example.com' where id = 1"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need delete-row on data/users" + ] + assert (await db.execute("select id, email from users order by id")).dicts() == [ + {"id": 1, "email": "a@example.com"}, + {"id": 2, "email": "b@example.com"}, + ] + + +@pytest.mark.asyncio +async def test_execute_write_update_requires_insert_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_update_requires_insert", name="data") + await db.execute_write("create table users (id integer primary key, name text)") + await db.execute_write("insert into users (id, name) values (1, 'Alice')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "update users set name = 'Alicia' where id = 1"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need insert-row on data/users" + ] + assert (await db.execute("select name from users where id = 1")).first()[0] == "Alice" + + @pytest.mark.asyncio async def test_execute_write_insert_select_requires_view_table_on_source(): ds = Datasette( @@ -1659,7 +1845,13 @@ async def test_execute_write_insert_select_requires_view_table_on_source(): "secret": { "permissions": {"view-table": {"id": "someone-else"}} }, - "public_log": {"permissions": {"insert-row": {"id": "writer"}}}, + "public_log": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + }, }, } } @@ -1740,6 +1932,8 @@ async def test_execute_write_rejects_function_operations(): "dogs": { "permissions": { "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, } } }, @@ -2117,6 +2311,8 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): "dogs": { "permissions": { "insert-row": {"id": "alice"}, + "update-row": {"id": "alice"}, + "delete-row": {"id": "alice"}, } } }, From 1932f8429fd3259d48fb848fdf893f9a004276e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:14:50 -0700 Subject: [PATCH 1800/1866] Deny user-authored schema table reads in write SQL Stop marking sqlite_master and sqlite_schema reads as internal as soon as the SQLite authorizer reports them. The later DDL-aware pass still treats schema catalog access as internal when it accompanies semantic CREATE, ALTER, or DROP operations. This makes explicit catalog reads in write SQL fall through to the deny-by-default path as unsupported read schema operations, preventing queries from copying private table definitions into writable tables. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 --- datasette/utils/sql_analysis.py | 1 - datasette/views/query_helpers.py | 4 +- tests/test_queries.py | 73 +++++++++++++++++++++++++++----- tests/test_utils_sql_analysis.py | 20 +++++++++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 8963da77..91216501 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -256,7 +256,6 @@ def analyze_sql_tables( target=arg1, source=source, column=column, - internal=target_type == "schema", ) return sqlite3.SQLITE_OK diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 7f3ef1bc..0e3d4e01 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -217,9 +217,7 @@ def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): permissions = permission_requirements_for_operation(operation) - required_permission = ", ".join( - permission.action for permission in permissions - ) + required_permission = ", ".join(permission.action for permission in permissions) rows.append( { "operation": operation.operation, diff --git a/tests/test_queries.py b/tests/test_queries.py index fcd19d1c..40bc5052 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1643,9 +1643,9 @@ async def test_execute_write_post_requires_database_and_table_permissions(): "Permission denied: need update-row on data/dogs" ] - ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ - "update-row" - ] = {"id": "writer"} + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"]["update-row"] = { + "id": "writer" + } missing_delete_permission = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1660,9 +1660,9 @@ async def test_execute_write_post_requires_database_and_table_permissions(): "Permission denied: need delete-row on data/dogs" ] - ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ - "delete-row" - ] = {"id": "writer"} + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"]["delete-row"] = { + "id": "writer" + } allowed = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1719,8 +1719,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): actor={"id": "writer"}, json={ "sql": ( - "insert or replace into users(id, email) " - "values (3, 'b@example.com')" + "insert or replace into users(id, email) " "values (3, 'b@example.com')" ) }, ) @@ -1773,7 +1772,9 @@ async def test_execute_write_update_or_replace_requires_delete_row_permission(): denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, - json={"sql": "update or replace users set email = 'b@example.com' where id = 1"}, + json={ + "sql": "update or replace users set email = 'b@example.com' where id = 1" + }, ) assert denied_response.status_code == 403 @@ -1826,7 +1827,9 @@ async def test_execute_write_update_requires_insert_row_permission(): assert denied_response.json()["errors"] == [ "Permission denied: need insert-row on data/users" ] - assert (await db.execute("select name from users where id = 1")).first()[0] == "Alice" + assert (await db.execute("select name from users where id = 1")).first()[ + 0 + ] == "Alice" @pytest.mark.asyncio @@ -1876,6 +1879,56 @@ async def test_execute_write_insert_select_requires_view_table_on_source(): assert (await db.execute("select value from public_log")).dicts() == [] +@pytest.mark.asyncio +async def test_execute_write_rejects_sqlite_master_reads(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + }, + "log": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + }, + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_sqlite_master_read", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("create table log (value text)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": ( + "insert into log " "select sql from sqlite_master where name = 'secret'" + ) + }, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: read schema" + ] + assert (await db.execute("select value from log")).dicts() == [] + + @pytest.mark.asyncio async def test_execute_write_create_table_as_select_requires_view_table_on_source(): ds = Datasette( diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 2ae11502..f931be51 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -65,6 +65,26 @@ def test_analyze_uses_sqlite_schema_as_default_database(conn): } +def test_analyze_user_schema_table_read_is_not_internal(conn): + analysis = analyze_sql_tables( + conn, + "insert into log select sql from sqlite_master where name = 'dogs'", + database_name="data", + ) + + assert { + "operation": "read", + "target_type": "schema", + "database": "data", + "sqlite_schema": "main", + "table": None, + "target": "sqlite_master", + "columns": ("name", "sql"), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + def operation_dict(operation): return { "operation": operation.operation, From 951f5a9f306ebe0bb8b3668ee698dc6cb6051d78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:30:05 -0700 Subject: [PATCH 1801/1866] Detect VACUUM in SQL analysis Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 --- datasette/stored_queries.py | 1 + datasette/utils/sql_analysis.py | 33 +++++++++++++++++++++++- tests/test_queries.py | 31 ++++++++++++++++++++++ tests/test_utils_sql_analysis.py | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index cf44a9ff..6746124a 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -720,6 +720,7 @@ def operation_is_write(operation: Operation) -> bool: "pragma", "analyze", "reindex", + "vacuum", "unknown", } diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 91216501..f2eb903f 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -22,6 +22,7 @@ SQLOperation = Literal[ "pragma", "analyze", "reindex", + "vacuum", "unknown", ] SQLTargetType = Literal[ @@ -423,10 +424,40 @@ def analyze_sql_tables( conn.set_authorizer(authorizer) try: - conn.execute("EXPLAIN " + sql, params if params is not None else {}).fetchall() + explain_rows = conn.execute( + "EXPLAIN " + sql, params if params is not None else {} + ).fetchall() finally: conn.set_authorizer(None) + if not operations: + vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) + if vacuum_row is not None: + schema_by_index = { + row[0]: row[1] for row in conn.execute("PRAGMA database_list") + } + sqlite_schema = schema_by_index.get(vacuum_row[2]) + database = database_for_schema(sqlite_schema) + record( + "vacuum", + "database", + database=database, + table=None, + sqlite_schema=sqlite_schema, + target=database, + source=None, + ) + else: + record( + "unknown", + "statement", + database=database_name, + table=None, + sqlite_schema=None, + target=None, + source=None, + ) + has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} diff --git a/tests/test_queries.py b/tests/test_queries.py index 40bc5052..bf371a80 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2011,6 +2011,37 @@ async def test_execute_write_rejects_function_operations(): assert (await db.execute("select name from dogs")).dicts() == [] +@pytest.mark.asyncio +async def test_execute_write_rejects_vacuum_operation(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_vacuum_operation", name="data") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "vacuum"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: vacuum database" + ] + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index f931be51..df4b3625 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -129,6 +129,50 @@ def test_analyze_create_table_operation(): ] +def test_analyze_vacuum_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables(conn, "vacuum", database_name="data") + finally: + conn.close() + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "vacuum", + "target_type": "database", + "database": "data", + "sqlite_schema": "main", + "table": None, + "target": "data", + "columns": (), + "source": None, + "internal": False, + } + ] + + +def test_analyze_statement_with_no_authorizer_callbacks_is_unknown(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables(conn, "reindex", database_name="data") + finally: + conn.close() + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "unknown", + "target_type": "statement", + "database": "data", + "sqlite_schema": None, + "table": None, + "target": None, + "columns": (), + "source": None, + "internal": False, + } + ] + + def test_analyze_transaction_operation(conn): analysis = analyze_sql_tables(conn, "commit", database_name="data") From 11bddc891918849e7c4a006c64d0217072aa499c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:51:12 -0700 Subject: [PATCH 1802/1866] Deny VACUUM in user-authored SQL Reject VACUUM explicitly during write-query permission analysis so arbitrary write SQL and untrusted stored write queries cannot run it, even when the actor has execute-write-sql. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 (P3) --- datasette/stored_queries.py | 16 ++++ datasette/views/database.py | 23 ++++- datasette/views/execute_write.py | 6 +- datasette/views/query_helpers.py | 9 +- tests/test_queries.py | 153 ++++++++++++++++++++++++++++++- 5 files changed, 199 insertions(+), 8 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index 6746124a..fd1cabf3 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -15,6 +15,13 @@ if TYPE_CHECKING: UNCHANGED = object() + +class QueryWriteRejected(Exception): + def __init__(self, message: str): + self.message = message + super().__init__(message) + + QUERY_OPTION_FIELDS = ( "hide_sql", "fragment", @@ -703,6 +710,12 @@ def operation_should_be_ignored(operation: Operation) -> bool: return operation.internal or operation.operation == "select" +def operation_forbidden_message(operation: Operation) -> str | None: + if operation.operation == "vacuum": + return "VACUUM is not allowed in user-supplied SQL" + return None + + def operation_is_write(operation: Operation) -> bool: return operation.operation in { "insert", @@ -746,6 +759,9 @@ async def ensure_query_write_permissions( for operation in analysis.operations: if operation_should_be_ignored(operation): continue + forbidden_message = operation_forbidden_message(operation) + if forbidden_message is not None: + raise QueryWriteRejected(forbidden_message) permissions = permission_requirements_for_operation(operation) if not permissions: raise Forbidden( diff --git a/datasette/views/database.py b/datasette/views/database.py index b558b002..ae1cf375 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import QueryWriteRejected, stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -453,9 +453,24 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - await _ensure_stored_query_execution_permissions( - datasette, db, stored_query, request.actor - ) + try: + await _ensure_stored_query_execution_permissions( + datasette, db, stored_query, request.actor + ) + except QueryWriteRejected as ex: + if request.headers.get("accept") == "application/json" or request.args.get( + "_json" + ): + return Response.json( + { + "ok": False, + "message": ex.message, + "redirect": None, + }, + status=403, + ) + datasette.add_message(request, ex.message, datasette.ERROR) + return Response.redirect(stored_query.on_error_redirect or request.path) # If database is immutable, return an error if not db.is_mutable: diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 19006ac5..57c4d78e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -163,13 +163,15 @@ class ExecuteWriteView(BaseView): except QueryValidationError as ex: if _wants_json(request, is_json, data): return _block_framing(_error([ex.message], ex.status)) + if ex.flash: + self.ds.add_message(request, ex.message, self.ds.ERROR) return await self._render_form( request, db, sql=sql or "", parameter_values=provided_params, - analysis_error=ex.message, - execution_message=ex.message, + analysis_error=None if ex.flash else ex.message, + execution_message=None if ex.flash else ex.message, execution_ok=False, status=ex.status, ) diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 0e3d4e01..92328ff3 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -3,6 +3,7 @@ import re from datasette.resources import DatabaseResource from datasette.stored_queries import ( + QueryWriteRejected, StoredQuery, operation_is_write, operation_should_be_ignored, @@ -47,9 +48,11 @@ _query_write_fields = { class QueryValidationError(Exception): - def __init__(self, message, status=400): + def __init__(self, message, status=400, *, flash=False): self.message = message self.status = status + self.flash = flash + super().__init__(message) def _actor_id(actor): @@ -194,6 +197,8 @@ async def _analyze_user_query(datasette, db, sql, *, actor): await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis ) + except QueryWriteRejected as ex: + raise QueryValidationError(ex.message, status=403, flash=True) from ex except Forbidden as ex: raise QueryValidationError(str(ex), status=403) from ex else: @@ -297,6 +302,8 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis ) + except QueryWriteRejected as ex: + raise QueryValidationError(ex.message, status=403, flash=True) from ex except Forbidden as ex: raise QueryValidationError(str(ex), status=403) from ex return parameter_names, params, analysis diff --git a/tests/test_queries.py b/tests/test_queries.py index bf371a80..b6e1637d 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2038,10 +2038,161 @@ async def test_execute_write_rejects_vacuum_operation(): assert denied_response.status_code == 403 assert denied_response.json()["errors"] == [ - "Unsupported SQL operation: vacuum database" + "VACUUM is not allowed in user-supplied SQL" ] +@pytest.mark.asyncio +async def test_execute_write_form_rejects_vacuum_operation_with_flash_error(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_vacuum_operation_form", name="data") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + data={"sql": "vacuum"}, + ) + + assert denied_response.status_code == 403 + assert ( + '

    VACUUM is not allowed in user-supplied SQL

    ' + in denied_response.text + ) + assert denied_response.text.count("VACUUM is not allowed in user-supplied SQL") == 1 + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_vacuum_operation(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("stored_query_vacuum_operation", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "vacuum_db", + "vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + denied_response = await ds.client.post( + "/data/vacuum_db?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert denied_response.status_code == 403 + assert "VACUUM is not allowed in user-supplied SQL" in denied_response.text + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_vacuum_operation_with_flash_error(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("stored_query_vacuum_operation_form", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "vacuum_db", + "vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + denied_response = await ds.client.post( + "/data/vacuum_db", + actor={"id": "writer"}, + data={}, + ) + + assert denied_response.status_code == 302 + assert denied_response.headers["location"] == "/data/vacuum_db" + assert ds.unsign(denied_response.cookies["ds_messages"], "messages") == [ + ["VACUUM is not allowed in user-supplied SQL", ds.ERROR] + ] + + +@pytest.mark.asyncio +async def test_trusted_stored_write_query_skips_vacuum_filtering(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("trusted_stored_query_vacuum", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "trusted_vacuum", + "vacuum", + is_write=True, + is_trusted=True, + source="config", + ) + + response = await ds.client.post( + "/data/trusted_vacuum?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( From 0c5053cdf64a0dc2d1e9808fa712b88233760512 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 17:26:50 -0700 Subject: [PATCH 1803/1866] Docs for //-/execute-write JSON API Closes #2750, refs #2742 --- docs/json_api.rst | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/docs/json_api.rst b/docs/json_api.rst index 48c70af6..fffc16d7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,6 +505,68 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _ExecuteWriteView: + +Executing write SQL +~~~~~~~~~~~~~~~~~~~ + +Actors with the :ref:`actions_execute_write_sql` permission can execute arbitrary writable SQL against a mutable database using ``/-/execute-write``. + +:: + + POST //-/execute-write + Content-Type: application/json + Authorization: Bearer dstok_ + +The request body must include a ``"sql"`` string. Named SQL parameters can be provided using the optional ``"params"`` object: + +.. code-block:: json + + { + "sql": "insert into dogs (name) values (:name)", + "params": { + "name": "Cleo" + } + } + +The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. + +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. + +A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: + +The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. + +.. code-block:: json + + { + "ok": true, + "message": "Query executed, 1 row affected", + "rowcount": 1, + "analysis": [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row, update-row, delete-row", + "source": null + } + ] + } + +If SQLite reports ``-1`` for the row count, the message will be ``"Query executed"``. + +Errors use the standard Datasette error format: + +.. code-block:: json + + { + "ok": false, + "errors": [ + "Permission denied: need execute-write-sql" + ] + } + .. _TableInsertView: Inserting rows From bcd989f4f8802a73a60c75f9bda77649c1347986 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 08:36:59 -0700 Subject: [PATCH 1804/1866] Detect and disallow insert to virtual/shadow table Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4565727978 --- datasette/stored_queries.py | 5 + datasette/utils/sql_analysis.py | 21 ++++- datasette/utils/sqlite.py | 112 ++++++++++++++++++++++ tests/test_queries.py | 153 +++++++++++++++++++++++++++++++ tests/test_utils.py | 45 ++++++++- tests/test_utils_sql_analysis.py | 47 ++++++++++ 6 files changed, 381 insertions(+), 2 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index fd1cabf3..b5aea221 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -713,6 +713,11 @@ def operation_should_be_ignored(operation: Operation) -> bool: def operation_forbidden_message(operation: Operation) -> str | None: if operation.operation == "vacuum": return "VACUUM is not allowed in user-supplied SQL" + if operation.operation in {"insert", "update", "delete"}: + if operation.table_kind == "virtual": + return "Writes to virtual tables are not allowed in user-supplied SQL" + if operation.table_kind == "shadow": + return "Writes to shadow tables are not allowed in user-supplied SQL" return None diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index f2eb903f..a71fa315 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Literal -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type SQLOperation = Literal[ "read", @@ -42,6 +42,7 @@ SQLTargetType = Literal[ SQLTableOperation = Literal["read", "insert", "update", "delete"] SQLSchemaOperation = Literal["create", "drop"] SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] +SQLTableKind = SQLiteTableType @dataclass(frozen=True) @@ -51,6 +52,7 @@ class Operation: database: str | None table: str | None sqlite_schema: str | None + table_kind: SQLTableKind | None = None target: str | None = None columns: tuple[str, ...] = () source: str | None = None @@ -500,6 +502,22 @@ def analyze_sql_tables( return True return False + table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + + def table_kind_for(key: OperationKey) -> SQLTableKind | None: + if ( + key.target_type != "table" + or key.operation not in {"read", "insert", "update", "delete"} + or key.table is None + ): + return None + cache_key = (key.sqlite_schema, key.table) + if cache_key not in table_kind_cache: + table_kind_cache[cache_key] = sqlite_table_type( + conn, key.table, schema=key.sqlite_schema + ) + return table_kind_cache[cache_key] + return SQLAnalysis( operations=tuple( Operation( @@ -508,6 +526,7 @@ def analyze_sql_tables( database=key.database, table=key.table, sqlite_schema=key.sqlite_schema, + table_kind=table_kind_for(key), target=key.target, columns=tuple(sorted(columns)), source=key.source, diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d0a2d783..130c5f62 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -1,3 +1,6 @@ +import re +from typing import Literal + using_pysqlite3 = False try: import pysqlite3 as sqlite3 @@ -10,6 +13,18 @@ if hasattr(sqlite3, "enable_callback_tracebacks"): sqlite3.enable_callback_tracebacks(True) _cached_sqlite_version = None +SQLiteTableType = Literal["table", "view", "virtual", "shadow"] +_VIRTUAL_TABLE_MODULE_RE = re.compile( + r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", + re.IGNORECASE, +) +_VIRTUAL_TABLE_SHADOW_SUFFIXES = { + "fts3": ("_content", "_segdir", "_segments", "_stat", "_docsize"), + "fts4": ("_content", "_segdir", "_segments", "_stat", "_docsize"), + "fts5": ("_data", "_idx", "_docsize", "_content", "_config"), + "rtree": ("_node", "_parent", "_rowid"), + "rtree_i32": ("_node", "_parent", "_rowid"), +} def sqlite_version(): @@ -36,5 +51,102 @@ def supports_table_xinfo(): return sqlite_version() >= (3, 26, 0) +def supports_table_list(): + return sqlite_version() >= (3, 37, 0) + + def supports_generated_columns(): return sqlite_version() >= (3, 31, 0) + + +def sqlite_table_type( + conn, + table: str, + *, + schema: str | None = "main", +) -> SQLiteTableType | None: + if supports_table_list(): + try: + query = "select type from pragma_table_list where name = ?" + params: tuple[str, ...] = (table,) + if schema is not None: + query += " and schema = ?" + params = (table, schema) + row = conn.execute(query, params).fetchone() + if row is not None and row[0] in {"table", "view", "virtual", "shadow"}: + return row[0] + except sqlite3.DatabaseError: + pass + return _sqlite_table_type_from_schema(conn, table, schema=schema) + + +def _sqlite_table_type_from_schema( + conn, + table: str, + *, + schema: str | None = "main", +) -> SQLiteTableType | None: + schema_table = _sqlite_schema_table(schema) + try: + row = conn.execute( + "select type, sql from {} where name = ?".format(schema_table), + (table,), + ).fetchone() + except sqlite3.DatabaseError: + return None + if row is None: + return None + object_type, sql = row + if object_type == "view": + return "view" + if object_type != "table": + return None + if _virtual_table_module(sql) is not None: + return "virtual" + if _is_known_shadow_table(conn, table, schema=schema): + return "shadow" + return "table" + + +def _is_known_shadow_table( + conn, + table: str, + *, + schema: str | None = "main", +) -> bool: + schema_table = _sqlite_schema_table(schema) + try: + rows = conn.execute( + "select name, sql from {} where type = 'table'".format(schema_table) + ).fetchall() + except sqlite3.DatabaseError: + return False + for virtual_table, sql in rows: + module = _virtual_table_module(sql) + if module is None: + continue + for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()): + if table == virtual_table + suffix: + return True + return False + + +def _sqlite_schema_table(schema: str | None) -> str: + if schema is None or schema == "main": + return "sqlite_master" + if schema == "temp": + return "sqlite_temp_master" + return "{}.sqlite_master".format(_quote_identifier(schema)) + + +def _quote_identifier(value: str) -> str: + return '"{}"'.format(value.replace('"', '""')) + + +def _virtual_table_module(sql: str | None) -> str | None: + if not sql: + return None + match = _VIRTUAL_TABLE_MODULE_RE.search(sql) + if match is None: + return None + return match.group(1).strip("\"'[]`").lower() diff --git a/tests/test_queries.py b/tests/test_queries.py index b6e1637d..73f8f3cf 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2193,6 +2193,159 @@ async def test_trusted_stored_write_query_skips_vacuum_filtering(): assert response.json()["ok"] is True +@pytest.mark.asyncio +async def test_execute_write_rejects_virtual_table_control_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_virtual_table_control", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs(docs) values('delete-all')"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to virtual tables are not allowed in user-supplied SQL" + ] + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_execute_write_rejects_regular_virtual_table_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_virtual_table_insert", name="data") + await db.execute_write("create virtual table docs using fts5(title, body)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs(rowid, title, body) values (1, 'a', 'b')"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to virtual tables are not allowed in user-supplied SQL" + ] + assert (await db.execute("select count(*) from docs")).first()[0] == 0 + + +@pytest.mark.asyncio +async def test_execute_write_rejects_shadow_table_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_shadow_table_insert", name="data") + await db.execute_write("create virtual table docs using fts5(title, body)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs_config(k, v) values ('x', 1)"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to shadow tables are not allowed in user-supplied SQL" + ] + assert (await db.execute("select count(*) from docs_config")).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_virtual_table_control_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("stored_query_virtual_table_control", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + await ds.add_query( + "data", + "delete_all_docs", + "insert into docs(docs) values('delete-all')", + is_write=True, + is_trusted=False, + source="user", + owner_id="root", + ) + + denied_response = await ds.client.post( + "/data/delete_all_docs?_json=1", + actor={"id": "root"}, + data={}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["message"] == ( + "Writes to virtual tables are not allowed in user-supplied SQL" + ) + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_trusted_stored_write_query_can_write_virtual_table(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("trusted_stored_query_virtual_table", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + await ds.add_query( + "data", + "trusted_delete_all", + "insert into docs(docs) values('delete-all')", + is_write=True, + is_trusted=True, + source="config", + ) + + response = await ds.client.post( + "/data/trusted_delete_all?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 0 + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( diff --git a/tests/test_utils.py b/tests/test_utils.py index 3fcb623e..e142bb5b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, sqlite_table_type import json import os import pathlib @@ -226,6 +226,49 @@ def test_detect_fts_different_table_names(table): conn.close() +@pytest.mark.parametrize("use_fallback", (False, True)) +def test_sqlite_table_type_detects_virtual_and_shadow_tables(monkeypatch, use_fallback): + if use_fallback: + monkeypatch.setattr("datasette.utils.sqlite.sqlite_version", lambda: (3, 25, 0)) + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table dogs(id integer primary key, name text); + create view dog_names as select name from dogs; + create virtual table search_index using fts5(title, body); + create virtual table boxes using rtree(id, minx, maxx, miny, maxy); + """) + + assert sqlite_table_type(conn, "dogs") == "table" + assert sqlite_table_type(conn, "dog_names") == "view" + assert sqlite_table_type(conn, "search_index") == "virtual" + assert sqlite_table_type(conn, "search_index_config") == "shadow" + assert sqlite_table_type(conn, "boxes") == "virtual" + assert sqlite_table_type(conn, "boxes_node") == "shadow" + assert sqlite_table_type(conn, "missing") is None + finally: + conn.close() + + +@pytest.mark.parametrize("use_fallback", (False, True)) +def test_sqlite_table_type_detects_attached_database_tables(monkeypatch, use_fallback): + if use_fallback: + monkeypatch.setattr("datasette.utils.sqlite.sqlite_version", lambda: (3, 25, 0)) + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + attach database ':memory:' as extra; + create table extra.cats(id integer primary key, name text); + create virtual table extra.cat_search using fts5(name); + """) + + assert sqlite_table_type(conn, "cats", schema="extra") == "table" + assert sqlite_table_type(conn, "cat_search", schema="extra") == "virtual" + assert sqlite_table_type(conn, "cat_search_data", schema="extra") == "shadow" + finally: + conn.close() + + @pytest.mark.parametrize( "url,expected", [ diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index df4b3625..979ff9e1 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -260,6 +260,53 @@ def test_analyze_create_virtual_table_operation(): } in [operation_dict(operation) for operation in analysis.operations] +def test_analyze_table_kind_for_regular_virtual_and_shadow_tables(): + conn = sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table dogs (id integer primary key, name text); + create virtual table docs using fts5(title, body, content=''); + """) + + regular_analysis = analyze_sql_tables( + conn, + "insert into dogs (name) values ('Cleo')", + database_name="data", + ) + virtual_analysis = analyze_sql_tables( + conn, + "insert into docs(docs) values('delete-all')", + database_name="data", + ) + shadow_analysis = analyze_sql_tables( + conn, + "insert into docs_config(k, v) values ('x', 1)", + database_name="data", + ) + finally: + conn.close() + + regular_insert = next( + operation + for operation in regular_analysis.operations + if operation.operation == "insert" and operation.table == "dogs" + ) + virtual_insert = next( + operation + for operation in virtual_analysis.operations + if operation.operation == "insert" and operation.table == "docs" + ) + shadow_insert = next( + operation + for operation in shadow_analysis.operations + if operation.operation == "insert" and operation.table == "docs_config" + ) + + assert regular_insert.table_kind == "table" + assert virtual_insert.table_kind == "virtual" + assert shadow_insert.table_kind == "shadow" + + def test_analyze_create_table_as_select_function_is_not_internal(): conn = sqlite3.connect(":memory:") try: From aaf00e9ec22b77e53f291ccedcbf2f499cce9e2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 08:42:06 -0700 Subject: [PATCH 1805/1866] Refactor hidden_table_names() to use new implemenatation Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4565727978 --- datasette/database.py | 80 +------------------------------- datasette/utils/sqlite.py | 29 ++++++++++++ tests/test_internals_database.py | 9 +--- tests/test_utils.py | 12 ++++- 4 files changed, 43 insertions(+), 87 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e7e9527e..10417670 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -26,7 +26,7 @@ from .utils import ( table_column_details, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_version +from .utils.sqlite import sqlite_hidden_table_names from .inspect import inspect_hash connections = threading.local() @@ -702,83 +702,7 @@ class Database: t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] - if sqlite_version()[1] >= 37: - hidden_tables += [x[0] for x in await self.execute(""" - with shadow_tables as ( - select name - from pragma_table_list - where [type] = 'shadow' - order by name - ), - core_tables as ( - select name - from sqlite_master - WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - combined as ( - select name from shadow_tables - union all - select name from core_tables - ) - select name from combined order by 1 - """)] - else: - hidden_tables += [x[0] for x in await self.execute(""" - WITH base AS ( - SELECT name - FROM sqlite_master - WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - fts_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config')) - ), - fts5_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%' - ), - fts5_shadow_tables AS ( - SELECT - printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name - FROM fts5_names - JOIN fts_suffixes - ), - fts3_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize')) - ), - fts3_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%' - OR sql LIKE '%VIRTUAL TABLE%USING FTS4%' - ), - fts3_shadow_tables AS ( - SELECT - printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name - FROM fts3_names - JOIN fts3_suffixes - ), - final AS ( - SELECT name FROM base - UNION ALL - SELECT name FROM fts5_shadow_tables - UNION ALL - SELECT name FROM fts3_shadow_tables - ) - SELECT name FROM final ORDER BY 1 - """)] - # Also hide any FTS tables that have a content= argument - hidden_tables += [x[0] for x in await self.execute(""" - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%' - AND sql LIKE '%USING FTS%' - AND sql LIKE '%content=%' - """)] + hidden_tables += await self.execute_fn(sqlite_hidden_table_names) has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 130c5f62..d3f52751 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -80,6 +80,28 @@ def sqlite_table_type( return _sqlite_table_type_from_schema(conn, table, schema=schema) +def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]: + schema_table = _sqlite_schema_table(schema) + try: + rows = conn.execute( + "select name, sql from {} where type = 'table'".format(schema_table) + ).fetchall() + except sqlite3.DatabaseError: + return [] + hidden_tables = [] + content_fts_tables = [] + for name, sql in rows: + if ( + name in {"sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"} + or name.startswith("_") + or sqlite_table_type(conn, name, schema=schema) == "shadow" + ): + hidden_tables.append(name) + elif _is_fts_content_virtual_table(sql): + content_fts_tables.append(name) + return sorted(hidden_tables) + content_fts_tables + + def _sqlite_table_type_from_schema( conn, table: str, @@ -150,3 +172,10 @@ def _virtual_table_module(sql: str | None) -> str | None: if match is None: return None return match.group(1).strip("\"'[]`").lower() + + +def _is_fts_content_virtual_table(sql: str | None) -> bool: + return ( + _virtual_table_module(sql) in {"fts3", "fts4", "fts5"} + and "content=" in sql.lower() + ) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index d6e130b4..88f9d571 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -8,7 +8,7 @@ from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.database import DatasetteClosedError from datasette.database import _deliver_write_result -from datasette.utils.sqlite import sqlite3, sqlite_version +from datasette.utils.sqlite import sqlite3 from datasette.utils import Column import pytest import time @@ -798,14 +798,7 @@ async def test_in_memory_databases_forbid_writes(app_client): assert await db.table_names() == ["foo"] -def pragma_table_list_supported(): - return sqlite_version()[1] >= 37 - - @pytest.mark.asyncio -@pytest.mark.skipif( - not pragma_table_list_supported(), reason="Requires PRAGMA table_list support" -) async def test_hidden_tables(app_client): ds = app_client.ds db = ds.add_database(Database(ds, is_memory=True, is_mutable=True)) diff --git a/tests/test_utils.py b/tests/test_utils.py index e142bb5b..90013537 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request -from datasette.utils.sqlite import sqlite3, sqlite_table_type +from datasette.utils.sqlite import sqlite3, sqlite_hidden_table_names, sqlite_table_type import json import os import pathlib @@ -246,6 +246,16 @@ def test_sqlite_table_type_detects_virtual_and_shadow_tables(monkeypatch, use_fa assert sqlite_table_type(conn, "boxes") == "virtual" assert sqlite_table_type(conn, "boxes_node") == "shadow" assert sqlite_table_type(conn, "missing") is None + assert sqlite_hidden_table_names(conn) == [ + "boxes_node", + "boxes_parent", + "boxes_rowid", + "search_index_config", + "search_index_content", + "search_index_data", + "search_index_docsize", + "search_index_idx", + ] finally: conn.close() From 2785fd29deef505f132902dcee86284e39e3fdcb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 09:03:10 -0700 Subject: [PATCH 1806/1866] Fix tests I just broke --- datasette/utils/sql_analysis.py | 86 +++++++++++++++++++-------------- datasette/utils/sqlite.py | 2 +- tests/test_utils.py | 14 ++++++ 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index a71fa315..b5d7ada8 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -193,6 +193,10 @@ _AUTHORIZER_ACTION_NAMES = { } +def _allow_authorizer_action(*args): + return sqlite3.SQLITE_OK + + def analyze_sql_tables( conn, sql: str, @@ -424,42 +428,59 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + conn.set_authorizer(authorizer) try: explain_rows = conn.execute( "EXPLAIN " + sql, params if params is not None else {} ).fetchall() + # Passing None before these lookups leaves a failing callback installed + # on Python 3.10, so use a permissive callback until they are complete. + conn.set_authorizer(_allow_authorizer_action) + + if not operations: + vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) + if vacuum_row is not None: + schema_by_index = { + row[0]: row[1] for row in conn.execute("PRAGMA database_list") + } + sqlite_schema = schema_by_index.get(vacuum_row[2]) + database = database_for_schema(sqlite_schema) + record( + "vacuum", + "database", + database=database, + table=None, + sqlite_schema=sqlite_schema, + target=database, + source=None, + ) + else: + record( + "unknown", + "statement", + database=database_name, + table=None, + sqlite_schema=None, + target=None, + source=None, + ) + + for key in operations: + if ( + key.target_type == "table" + and key.operation in {"read", "insert", "update", "delete"} + and key.table is not None + ): + cache_key = (key.sqlite_schema, key.table) + if cache_key not in table_kind_cache: + table_kind_cache[cache_key] = sqlite_table_type( + conn, key.table, schema=key.sqlite_schema + ) finally: conn.set_authorizer(None) - if not operations: - vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) - if vacuum_row is not None: - schema_by_index = { - row[0]: row[1] for row in conn.execute("PRAGMA database_list") - } - sqlite_schema = schema_by_index.get(vacuum_row[2]) - database = database_for_schema(sqlite_schema) - record( - "vacuum", - "database", - database=database, - table=None, - sqlite_schema=sqlite_schema, - target=database, - source=None, - ) - else: - record( - "unknown", - "statement", - database=database_name, - table=None, - sqlite_schema=None, - target=None, - source=None, - ) - has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} @@ -502,8 +523,6 @@ def analyze_sql_tables( return True return False - table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} - def table_kind_for(key: OperationKey) -> SQLTableKind | None: if ( key.target_type != "table" @@ -511,12 +530,7 @@ def analyze_sql_tables( or key.table is None ): return None - cache_key = (key.sqlite_schema, key.table) - if cache_key not in table_kind_cache: - table_kind_cache[cache_key] = sqlite_table_type( - conn, key.table, schema=key.sqlite_schema - ) - return table_kind_cache[cache_key] + return table_kind_cache[(key.sqlite_schema, key.table)] return SQLAnalysis( operations=tuple( diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d3f52751..5a7c6c38 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -16,7 +16,7 @@ _cached_sqlite_version = None SQLiteTableType = Literal["table", "view", "virtual", "shadow"] _VIRTUAL_TABLE_MODULE_RE = re.compile( r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", - re.IGNORECASE, + re.IGNORECASE | re.DOTALL, ) _VIRTUAL_TABLE_SHADOW_SUFFIXES = { "fts3": ("_content", "_segdir", "_segments", "_stat", "_docsize"), diff --git a/tests/test_utils.py b/tests/test_utils.py index 90013537..e83eed7a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -279,6 +279,20 @@ def test_sqlite_table_type_detects_attached_database_tables(monkeypatch, use_fal conn.close() +def test_sqlite_hidden_table_names_hides_multiline_content_fts_table(): + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table searchable(id integer primary key, body text); + create virtual table searchable_fts + using fts5(body, content='searchable', content_rowid='id'); + """) + + assert "searchable_fts" in sqlite_hidden_table_names(conn) + finally: + conn.close() + + @pytest.mark.parametrize( "url,expected", [ From 8bd7e165f465fe057beace2b17d52c0a347819f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 09:50:56 -0700 Subject: [PATCH 1807/1866] Refactored for code readability --- datasette/app.py | 5 +- datasette/stored_queries.py | 211 +------------------------ datasette/views/database.py | 3 +- datasette/views/query_helpers.py | 27 ++-- datasette/write_sql.py | 255 +++++++++++++++++++++++++++++++ tests/test_write_sql.py | 59 +++++++ 6 files changed, 339 insertions(+), 221 deletions(-) create mode 100644 datasette/write_sql.py create mode 100644 tests/test_write_sql.py diff --git a/datasette/app.py b/datasette/app.py index 56b89789..e7f34e69 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,7 +42,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType -from . import stored_queries +from . import stored_queries, write_sql from .views import Context from .views.database import ( database_download, @@ -1197,7 +1197,8 @@ class Datasette: async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None ): - return await stored_queries.ensure_query_write_permissions( + # Raise Forbidden or QueryWriteRejected if SQL should not run + return await write_sql.ensure_query_write_permissions( self, database, sql, actor=actor, params=params, analysis=analysis ) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index b5aea221..b6ac49b8 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -2,26 +2,13 @@ from __future__ import annotations from dataclasses import dataclass import json -from typing import Any, Iterable, TYPE_CHECKING +from typing import Any, Iterable -from .resources import DatabaseResource, TableResource -from .permissions import Resource -from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components -from .utils.asgi import Forbidden -from .utils.sql_analysis import Operation, SQLAnalysis - -if TYPE_CHECKING: - from .app import Datasette +from .utils import tilde_encode, urlsafe_components UNCHANGED = object() -class QueryWriteRejected(Exception): - def __init__(self, message: str): - self.message = message - super().__init__(message) - - QUERY_OPTION_FIELDS = ( "hide_sql", "fragment", @@ -593,197 +580,3 @@ async def list_queries( has_more=has_more, limit=limit, ) - - -@dataclass(frozen=True) -class PermissionRequirement: - action: str - resource: Resource - - -def row_mutation_requirements( - database: str, table: str -) -> tuple[PermissionRequirement, ...]: - resource = TableResource(database=database, table=table) - return tuple( - PermissionRequirement(action=action, resource=resource) - for action in ("insert-row", "update-row", "delete-row") - ) - - -def permission_requirements_for_operation( - operation: Operation, -) -> tuple[PermissionRequirement, ...]: - if ( - operation.operation == "read" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="view-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation in {"insert", "update"} - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return row_mutation_requirements( - database=operation.database, - table=operation.table, - ) - if ( - operation.operation == "delete" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="delete-row", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if operation.operation == "create" and operation.target_type == "table": - if operation.database is None: - return () - return ( - PermissionRequirement( - action="create-table", - resource=DatabaseResource(database=operation.database), - ), - ) - if ( - operation.operation == "alter" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="alter-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation == "drop" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="drop-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation in {"create", "drop"} - and operation.target_type == "index" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="alter-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - return () - - -def operation_should_be_ignored(operation: Operation) -> bool: - return operation.internal or operation.operation == "select" - - -def operation_forbidden_message(operation: Operation) -> str | None: - if operation.operation == "vacuum": - return "VACUUM is not allowed in user-supplied SQL" - if operation.operation in {"insert", "update", "delete"}: - if operation.table_kind == "virtual": - return "Writes to virtual tables are not allowed in user-supplied SQL" - if operation.table_kind == "shadow": - return "Writes to shadow tables are not allowed in user-supplied SQL" - return None - - -def operation_is_write(operation: Operation) -> bool: - return operation.operation in { - "insert", - "update", - "delete", - "create", - "alter", - "drop", - "begin", - "commit", - "rollback", - "savepoint", - "attach", - "detach", - "pragma", - "analyze", - "reindex", - "vacuum", - "unknown", - } - - -async def ensure_query_write_permissions( - datasette: Datasette, - database: str, - sql: str, - *, - actor: dict[str, object] | None = None, - params: dict[str, object] | None = None, - analysis: SQLAnalysis | None = None, -) -> SQLAnalysis: - db = datasette.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for operation in analysis.operations: - if operation_should_be_ignored(operation): - continue - forbidden_message = operation_forbidden_message(operation) - if forbidden_message is not None: - raise QueryWriteRejected(forbidden_message) - permissions = permission_requirements_for_operation(operation) - if not permissions: - raise Forbidden( - "Unsupported SQL operation: {} {}".format( - operation.operation, operation.target_type - ) - ) - if operation.database != database: - raise Forbidden("Writable queries may not access attached databases") - for permission in permissions: - if not await datasette.allowed( - action=permission.action, - resource=permission.resource, - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {permission.action} " - f"on {permission.resource}" - ) - return analysis diff --git a/datasette/views/database.py b/datasette/views/database.py index ae1cf375..b4a964f1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,8 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import QueryWriteRejected, stored_query_to_dict +from datasette.stored_queries import stored_query_to_dict +from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, await_me_maybe, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 92328ff3..712832e8 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -3,11 +3,14 @@ import re from datasette.resources import DatabaseResource from datasette.stored_queries import ( - QueryWriteRejected, StoredQuery, +) +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + QueryWriteRejected, + RequireWriteSqlPermissions, + decision_for_write_sql_operation, operation_is_write, - operation_should_be_ignored, - permission_requirements_for_operation, ) from datasette.utils import ( named_parameters as derive_named_parameters, @@ -212,7 +215,9 @@ async def _analyze_user_query(datasette, db, sql, *, actor): def _display_operations(analysis: SQLAnalysis) -> list[Operation]: operations = [] for operation in analysis.operations: - if operation_should_be_ignored(operation): + if isinstance( + decision_for_write_sql_operation(operation), IgnoreWriteSqlOperation + ): continue operations.append(operation) return operations @@ -221,8 +226,12 @@ def _display_operations(analysis: SQLAnalysis) -> list[Operation]: def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): - permissions = permission_requirements_for_operation(operation) - required_permission = ", ".join(permission.action for permission in permissions) + decision = decision_for_write_sql_operation(operation) + required_permission = ( + ", ".join(permission.action for permission in decision.permissions) + if isinstance(decision, RequireWriteSqlPermissions) + else "" + ) rows.append( { "operation": operation.operation, @@ -241,10 +250,10 @@ async def _analysis_rows_with_permissions( rows = _analysis_rows(analysis) is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): - permissions = permission_requirements_for_operation(operation) - if permissions: + decision = decision_for_write_sql_operation(operation) + if isinstance(decision, RequireWriteSqlPermissions): row["allowed"] = True - for permission in permissions: + for permission in decision.permissions: if not await datasette.allowed( action=permission.action, resource=permission.resource, diff --git a/datasette/write_sql.py b/datasette/write_sql.py new file mode 100644 index 00000000..2e1b69af --- /dev/null +++ b/datasette/write_sql.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from .permissions import Resource +from .resources import DatabaseResource, TableResource +from .utils import named_parameters, sqlite3 +from .utils.asgi import Forbidden +from .utils.sql_analysis import Operation, SQLAnalysis + +if TYPE_CHECKING: + from .app import Datasette + + +class QueryWriteRejected(Exception): + def __init__(self, message: str): + self.message = message + super().__init__(message) + + +@dataclass(frozen=True) +class PermissionRequirement: + action: str + resource: Resource + + +PermissionRequirements = tuple[PermissionRequirement, ...] + + +class WriteSqlOperationDecision: + """What Datasette should do with one operation in user-supplied write SQL.""" + + +@dataclass(frozen=True) +class IgnoreWriteSqlOperation(WriteSqlOperationDecision): + reason: str + + +@dataclass(frozen=True) +class RequireWriteSqlPermissions(WriteSqlOperationDecision): + permissions: PermissionRequirements + + +@dataclass(frozen=True) +class RejectWriteSqlOperation(WriteSqlOperationDecision): + message: str + + +@dataclass(frozen=True) +class UnsupportedWriteSqlOperation(WriteSqlOperationDecision): + message: str + + +def row_mutation_requirements(database: str, table: str) -> PermissionRequirements: + resource = TableResource(database=database, table=table) + return tuple( + PermissionRequirement(action=action, resource=resource) + for action in ("insert-row", "update-row", "delete-row") + ) + + +def decision_for_write_sql_operation( + operation: Operation, +) -> WriteSqlOperationDecision: + unsupported_message = ( + f"Unsupported SQL operation: {operation.operation} {operation.target_type}" + ) + if operation.internal: + return IgnoreWriteSqlOperation("internal SQLite operation") + if operation.operation == "select": + return IgnoreWriteSqlOperation("select statement") + if operation.operation == "vacuum": + return RejectWriteSqlOperation("VACUUM is not allowed in user-supplied SQL") + if operation.operation in {"insert", "update", "delete"}: + if operation.table_kind == "virtual": + return RejectWriteSqlOperation( + "Writes to virtual tables are not allowed in user-supplied SQL" + ) + if operation.table_kind == "shadow": + return RejectWriteSqlOperation( + "Writes to shadow tables are not allowed in user-supplied SQL" + ) + if operation.operation == "function": + # SQL functions currently have no Datasette permission mapping. They are + # rejected by the user-supplied write SQL allow-list as unsupported. + return UnsupportedWriteSqlOperation(unsupported_message) + if ( + operation.operation == "read" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="view-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation in {"insert", "update"} + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + row_mutation_requirements( + database=operation.database, + table=operation.table, + ) + ) + if ( + operation.operation == "delete" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="delete-row", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if operation.operation == "create" and operation.target_type == "table": + if operation.database is None: + return UnsupportedWriteSqlOperation(unsupported_message) + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="create-table", + resource=DatabaseResource(database=operation.database), + ), + ) + ) + if ( + operation.operation == "alter" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation == "drop" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="drop-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation in {"create", "drop"} + and operation.target_type == "index" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + return UnsupportedWriteSqlOperation(unsupported_message) + + +def operation_is_write(operation: Operation) -> bool: + return operation.operation in { + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "savepoint", + "attach", + "detach", + "pragma", + "analyze", + "reindex", + "vacuum", + "unknown", + } + + +async def ensure_query_write_permissions( + datasette: Datasette, + database: str, + sql: str, + *, + actor: dict[str, object] | None = None, + params: dict[str, object] | None = None, + analysis: SQLAnalysis | None = None, +) -> SQLAnalysis: + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for operation in analysis.operations: + decision = decision_for_write_sql_operation(operation) + if isinstance(decision, IgnoreWriteSqlOperation): + continue + if isinstance(decision, RejectWriteSqlOperation): + raise QueryWriteRejected(decision.message) + if isinstance(decision, UnsupportedWriteSqlOperation): + raise Forbidden(decision.message) + permissions = decision.permissions + if operation.database != database: + raise Forbidden("Writable queries may not access attached databases") + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {permission.action} " + f"on {permission.resource}" + ) + return analysis diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py new file mode 100644 index 00000000..cfaf0f53 --- /dev/null +++ b/tests/test_write_sql.py @@ -0,0 +1,59 @@ +from datasette.utils.sql_analysis import Operation +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + RejectWriteSqlOperation, + RequireWriteSqlPermissions, + UnsupportedWriteSqlOperation, + WriteSqlOperationDecision, + decision_for_write_sql_operation, +) + + +def test_decision_for_write_sql_operation_ignores_internal_and_select_operations(): + internal_decision = decision_for_write_sql_operation( + Operation("read", "schema", None, None, "main", internal=True) + ) + select_decision = decision_for_write_sql_operation( + Operation("select", "statement", None, None, None) + ) + + assert isinstance(internal_decision, IgnoreWriteSqlOperation) + assert isinstance(internal_decision, WriteSqlOperationDecision) + assert isinstance(select_decision, IgnoreWriteSqlOperation) + assert isinstance(select_decision, WriteSqlOperationDecision) + + +def test_decision_for_write_sql_operation_requires_table_write_permissions(): + decision = decision_for_write_sql_operation( + Operation("insert", "table", "data", "dogs", None) + ) + + assert isinstance(decision, RequireWriteSqlPermissions) + assert [permission.action for permission in decision.permissions] == [ + "insert-row", + "update-row", + "delete-row", + ] + assert [str(permission.resource) for permission in decision.permissions] == [ + "data/dogs", + "data/dogs", + "data/dogs", + ] + + +def test_decision_for_write_sql_operation_rejects_vacuum(): + decision = decision_for_write_sql_operation( + Operation("vacuum", "statement", None, None, None) + ) + + assert isinstance(decision, RejectWriteSqlOperation) + assert decision.message == "VACUUM is not allowed in user-supplied SQL" + + +def test_decision_for_write_sql_operation_reports_unsupported_functions(): + decision = decision_for_write_sql_operation( + Operation("function", "function", None, None, None, target="upper") + ) + + assert isinstance(decision, UnsupportedWriteSqlOperation) + assert decision.message == "Unsupported SQL operation: function function" From 51dab16149f8b345d46cf517fa03b95fc1028234 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 10:22:16 -0700 Subject: [PATCH 1808/1866] Allow SQL functions in SQL write queries Closes #2751 --- datasette/write_sql.py | 4 +- docs/authentication.rst | 2 +- docs/json_api.rst | 2 +- docs/sql_queries.rst | 2 +- tests/test_queries.py | 83 +++++++++++++++++++++++++++++++++++++---- tests/test_write_sql.py | 13 ++++++- 6 files changed, 91 insertions(+), 15 deletions(-) diff --git a/datasette/write_sql.py b/datasette/write_sql.py index 2e1b69af..cdc0c6d3 100644 --- a/datasette/write_sql.py +++ b/datasette/write_sql.py @@ -82,9 +82,7 @@ def decision_for_write_sql_operation( "Writes to shadow tables are not allowed in user-supplied SQL" ) if operation.operation == "function": - # SQL functions currently have no Datasette permission mapping. They are - # rejected by the user-supplied write SQL allow-list as unsupported. - return UnsupportedWriteSqlOperation(unsupported_message) + return IgnoreWriteSqlOperation("SQL function") if ( operation.operation == "read" and operation.target_type == "table" diff --git a/docs/authentication.rst b/docs/authentication.rst index f720c12f..a0891900 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1425,7 +1425,7 @@ See also :ref:`the default_allow_sql setting `. execute-write-sql ----------------- -Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. +Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/json_api.rst b/docs/json_api.rst index fffc16d7..d502299e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -531,7 +531,7 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. -Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. SQL functions are allowed and are not separately restricted by Datasette permissions. A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index f593a534..d427ea2b 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -140,7 +140,7 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. -Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: .. _trusted_saved_queries: diff --git a/tests/test_queries.py b/tests/test_queries.py index 73f8f3cf..9c3ebcc8 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1414,6 +1414,11 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): actor={"id": "root"}, params={"sql": "insert into dogs (name) values (:name)"}, ) + function_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "root"}, + params={"sql": "insert into dogs (name) values (upper(:name))"}, + ) read_only_response = await ds.client.get( "/data/-/execute-write/analyze", actor={"id": "root"}, @@ -1438,6 +1443,22 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): ] assert "params" not in data + assert function_response.status_code == 200 + function_data = function_response.json() + assert function_data["ok"] is True + assert function_data["parameters"] == ["name"] + assert function_data["execute_disabled"] is False + assert function_data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row, update-row, delete-row", + "source": None, + "allowed": True, + } + ] + assert read_only_response.status_code == 200 read_only_data = read_only_response.json() assert read_only_data["ok"] is False @@ -1970,7 +1991,7 @@ async def test_execute_write_create_table_as_select_requires_view_table_on_sourc @pytest.mark.asyncio -async def test_execute_write_rejects_function_operations(): +async def test_execute_write_allows_function_operations(): ds = Datasette( memory=True, default_deny=True, @@ -1998,17 +2019,65 @@ async def test_execute_write_rejects_function_operations(): await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() - denied_response = await ds.client.post( + response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, json={"sql": "insert into dogs (name) values (upper('cleo'))"}, ) - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Unsupported SQL operation: function function" - ] - assert (await db.execute("select name from dogs")).dicts() == [] + assert response.status_code == 200 + assert response.json()["ok"] is True + assert (await db.execute("select name from dogs")).dicts() == [{"name": "CLEO"}] + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_allows_function_operations(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("stored_query_function_operation", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (upper(:name))", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + response = await ds.client.post( + "/data/insert_dog?_json=1", + actor={"id": "writer"}, + data={"name": "cleo"}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert (await db.execute("select name from dogs")).dicts() == [{"name": "CLEO"}] @pytest.mark.asyncio diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py index cfaf0f53..6d95c3c4 100644 --- a/tests/test_write_sql.py +++ b/tests/test_write_sql.py @@ -50,10 +50,19 @@ def test_decision_for_write_sql_operation_rejects_vacuum(): assert decision.message == "VACUUM is not allowed in user-supplied SQL" -def test_decision_for_write_sql_operation_reports_unsupported_functions(): +def test_decision_for_write_sql_operation_ignores_functions(): decision = decision_for_write_sql_operation( Operation("function", "function", None, None, None, target="upper") ) + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "SQL function" + + +def test_decision_for_write_sql_operation_reports_unsupported_operations(): + decision = decision_for_write_sql_operation( + Operation("unknown", "unknown", None, None, None) + ) + assert isinstance(decision, UnsupportedWriteSqlOperation) - assert decision.message == "Unsupported SQL operation: function function" + assert decision.message == "Unsupported SQL operation: unknown unknown" From b2b20b36c52ea446fb05fe688b636b83d187e6a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 10:24:40 -0700 Subject: [PATCH 1809/1866] Document write SQL analyzer restrictions Expand the unreleased changelog with the deny-by-default operation analysis model, SQL function handling, and the VACUUM and virtual/shadow table restrictions for user-supplied write SQL. Clarify the /-/execute-write JSON API documentation with the same restrictions and DDL permission requirements. --- docs/changelog.rst | 2 ++ docs/json_api.rst | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ba713ee..a4be98b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Write SQL UI - New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`) - Added the new :ref:`execute-write-sql ` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row `, :ref:`update-row ` and :ref:`delete-row `, and writes to attached databases are rejected. (:issue:`2742`) +- The write SQL analyzer now uses a deny-by-default model for unsupported operations. Reads from source tables require :ref:`view-table ` permission, schema changes require :ref:`create-table `, :ref:`alter-table ` or :ref:`drop-table ` as appropriate, and row mutation statements require the full ``insert-row``, ``update-row`` and ``delete-row`` permission set. SQL functions are allowed and are not separately permission-gated. (:issue:`2748`) +- User-supplied write SQL now rejects ``VACUUM`` and writes to SQLite virtual tables or shadow tables. These restrictions also apply to untrusted stored write queries; trusted configured stored queries continue to skip these filters. (:issue:`2748`) Plugin API changes ~~~~~~~~~~~~~~~~~~ diff --git a/docs/json_api.rst b/docs/json_api.rst index d502299e..db19afc2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -531,7 +531,9 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. -Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. SQL functions are allowed and are not separately restricted by Datasette permissions. +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. Schema changes require ``create-table``, ``alter-table`` or ``drop-table`` permissions as appropriate. + +Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in arbitrary write SQL, and writes to SQLite virtual tables or shadow tables are rejected. SQL functions are allowed and are not separately restricted by Datasette permissions. A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: From cbe9594a3dcac1f91a6baa7ac99a138c22a71a8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 11:00:04 -0700 Subject: [PATCH 1810/1866] Use SQLiteTableType directly in SQL analysis Remove the redundant SQLTableKind alias from the write SQL analysis model. Operation.table_kind and the analyzer cache now use the SQLite metadata classification type directly, making the source of table-kind values clearer. --- datasette/utils/sql_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index b5d7ada8..0a3a947c 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -42,7 +42,6 @@ SQLTargetType = Literal[ SQLTableOperation = Literal["read", "insert", "update", "delete"] SQLSchemaOperation = Literal["create", "drop"] SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] -SQLTableKind = SQLiteTableType @dataclass(frozen=True) @@ -52,7 +51,7 @@ class Operation: database: str | None table: str | None sqlite_schema: str | None - table_kind: SQLTableKind | None = None + table_kind: SQLiteTableType | None = None target: str | None = None columns: tuple[str, ...] = () source: str | None = None @@ -428,7 +427,7 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK - table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + table_kind_cache: dict[tuple[str | None, str], SQLiteTableType | None] = {} conn.set_authorizer(authorizer) try: @@ -523,7 +522,7 @@ def analyze_sql_tables( return True return False - def table_kind_for(key: OperationKey) -> SQLTableKind | None: + def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( key.target_type != "table" or key.operation not in {"read", "insert", "update", "delete"} From 17f45b884b4b4844e9f0cce0fef402e888c690f0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 12:06:57 -0700 Subject: [PATCH 1811/1866] Clarify ignored write SQL operation tests Split the combined ignored-operation decision test into separate internal-operation and select-statement cases. Assert the decision reason for each case instead of checking the shared base class, so the tests document why those operations are ignored. --- tests/test_write_sql.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py index 6d95c3c4..75d6b6e1 100644 --- a/tests/test_write_sql.py +++ b/tests/test_write_sql.py @@ -4,23 +4,26 @@ from datasette.write_sql import ( RejectWriteSqlOperation, RequireWriteSqlPermissions, UnsupportedWriteSqlOperation, - WriteSqlOperationDecision, decision_for_write_sql_operation, ) -def test_decision_for_write_sql_operation_ignores_internal_and_select_operations(): - internal_decision = decision_for_write_sql_operation( +def test_decision_for_write_sql_operation_ignores_internal_operations(): + decision = decision_for_write_sql_operation( Operation("read", "schema", None, None, "main", internal=True) ) - select_decision = decision_for_write_sql_operation( + + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "internal SQLite operation" + + +def test_decision_for_write_sql_operation_ignores_select_statement_operations(): + decision = decision_for_write_sql_operation( Operation("select", "statement", None, None, None) ) - assert isinstance(internal_decision, IgnoreWriteSqlOperation) - assert isinstance(internal_decision, WriteSqlOperationDecision) - assert isinstance(select_decision, IgnoreWriteSqlOperation) - assert isinstance(select_decision, WriteSqlOperationDecision) + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "select statement" def test_decision_for_write_sql_operation_requires_table_write_permissions(): From 0b7c26c6c8bf4827c02aba9707b1db0eb63aeaa5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 12:09:02 -0700 Subject: [PATCH 1812/1866] Refactored write decision tests --- tests/test_write_sql.py | 71 ---------------- tests/test_write_sql_operation_decisions.py | 94 +++++++++++++++++++++ 2 files changed, 94 insertions(+), 71 deletions(-) delete mode 100644 tests/test_write_sql.py create mode 100644 tests/test_write_sql_operation_decisions.py diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py deleted file mode 100644 index 75d6b6e1..00000000 --- a/tests/test_write_sql.py +++ /dev/null @@ -1,71 +0,0 @@ -from datasette.utils.sql_analysis import Operation -from datasette.write_sql import ( - IgnoreWriteSqlOperation, - RejectWriteSqlOperation, - RequireWriteSqlPermissions, - UnsupportedWriteSqlOperation, - decision_for_write_sql_operation, -) - - -def test_decision_for_write_sql_operation_ignores_internal_operations(): - decision = decision_for_write_sql_operation( - Operation("read", "schema", None, None, "main", internal=True) - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "internal SQLite operation" - - -def test_decision_for_write_sql_operation_ignores_select_statement_operations(): - decision = decision_for_write_sql_operation( - Operation("select", "statement", None, None, None) - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "select statement" - - -def test_decision_for_write_sql_operation_requires_table_write_permissions(): - decision = decision_for_write_sql_operation( - Operation("insert", "table", "data", "dogs", None) - ) - - assert isinstance(decision, RequireWriteSqlPermissions) - assert [permission.action for permission in decision.permissions] == [ - "insert-row", - "update-row", - "delete-row", - ] - assert [str(permission.resource) for permission in decision.permissions] == [ - "data/dogs", - "data/dogs", - "data/dogs", - ] - - -def test_decision_for_write_sql_operation_rejects_vacuum(): - decision = decision_for_write_sql_operation( - Operation("vacuum", "statement", None, None, None) - ) - - assert isinstance(decision, RejectWriteSqlOperation) - assert decision.message == "VACUUM is not allowed in user-supplied SQL" - - -def test_decision_for_write_sql_operation_ignores_functions(): - decision = decision_for_write_sql_operation( - Operation("function", "function", None, None, None, target="upper") - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "SQL function" - - -def test_decision_for_write_sql_operation_reports_unsupported_operations(): - decision = decision_for_write_sql_operation( - Operation("unknown", "unknown", None, None, None) - ) - - assert isinstance(decision, UnsupportedWriteSqlOperation) - assert decision.message == "Unsupported SQL operation: unknown unknown" diff --git a/tests/test_write_sql_operation_decisions.py b/tests/test_write_sql_operation_decisions.py new file mode 100644 index 00000000..cc19f701 --- /dev/null +++ b/tests/test_write_sql_operation_decisions.py @@ -0,0 +1,94 @@ +import pytest + +from datasette.utils.sql_analysis import Operation +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + RejectWriteSqlOperation, + RequireWriteSqlPermissions, + UnsupportedWriteSqlOperation, + decision_for_write_sql_operation, +) + + +@pytest.mark.parametrize( + ("operation", "reason"), + ( + pytest.param( + Operation("read", "schema", None, None, "main", internal=True), + "internal SQLite operation", + id="internal", + ), + pytest.param( + Operation("select", "statement", None, None, None), + "select statement", + id="select-statement", + ), + pytest.param( + Operation("function", "function", None, None, None, target="upper"), + "SQL function", + id="function", + ), + ), +) +def test_decision_for_write_sql_operation_ignores_operations(operation, reason): + decision = decision_for_write_sql_operation(operation) + + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == reason + + +@pytest.mark.parametrize("operation", ("insert", "update")) +def test_decision_for_write_sql_operation_requires_table_write_permissions(operation): + decision = decision_for_write_sql_operation( + Operation(operation, "table", "data", "dogs", None) + ) + + assert isinstance(decision, RequireWriteSqlPermissions) + assert [permission.action for permission in decision.permissions] == [ + "insert-row", + "update-row", + "delete-row", + ] + assert [str(permission.resource) for permission in decision.permissions] == [ + "data/dogs", + "data/dogs", + "data/dogs", + ] + + +@pytest.mark.parametrize( + ("operation", "message"), + ( + pytest.param( + Operation("vacuum", "statement", None, None, None), + "VACUUM is not allowed in user-supplied SQL", + id="vacuum", + ), + pytest.param( + Operation("insert", "table", "data", "docs", None, table_kind="virtual"), + "Writes to virtual tables are not allowed in user-supplied SQL", + id="virtual-table", + ), + pytest.param( + Operation( + "insert", "table", "data", "docs_data", None, table_kind="shadow" + ), + "Writes to shadow tables are not allowed in user-supplied SQL", + id="shadow-table", + ), + ), +) +def test_decision_for_write_sql_operation_rejects_operations(operation, message): + decision = decision_for_write_sql_operation(operation) + + assert isinstance(decision, RejectWriteSqlOperation) + assert decision.message == message + + +def test_decision_for_write_sql_operation_reports_unsupported_operations(): + decision = decision_for_write_sql_operation( + Operation("unknown", "unknown", None, None, None) + ) + + assert isinstance(decision, UnsupportedWriteSqlOperation) + assert decision.message == "Unsupported SQL operation: unknown unknown" From cd838daef4d066e584b047164d8e2a5e96909511 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:22:21 -0700 Subject: [PATCH 1813/1866] Refactor tests a bit --- tests/test_queries.py | 449 +++++++++++++++++++++--------------------- 1 file changed, 225 insertions(+), 224 deletions(-) diff --git a/tests/test_queries.py b/tests/test_queries.py index 9c3ebcc8..216cb211 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1700,8 +1700,22 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.parametrize( + "database_name, sql", + ( + ( + "execute_write_insert_or_replace", + "insert or replace into users(id, email) values (3, 'b@example.com')", + ), + ( + "execute_write_update_or_replace", + "update or replace users set email = 'b@example.com' where id = 1", + ), + ), + ids=("insert-or-replace", "update-or-replace"), +) @pytest.mark.asyncio -async def test_execute_write_insert_or_replace_requires_delete_row_permission(): +async def test_execute_write_replace_requires_delete_row_permission(database_name, sql): ds = Datasette( memory=True, default_deny=True, @@ -1725,7 +1739,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): } }, ) - db = ds.add_memory_database("execute_write_insert_or_replace", name="data") + db = ds.add_memory_database(database_name, name="data") await db.execute_write( "create table users (id integer primary key, email text unique)" ) @@ -1738,64 +1752,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, - json={ - "sql": ( - "insert or replace into users(id, email) " "values (3, 'b@example.com')" - ) - }, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Permission denied: need delete-row on data/users" - ] - assert (await db.execute("select id, email from users order by id")).dicts() == [ - {"id": 1, "email": "a@example.com"}, - {"id": 2, "email": "b@example.com"}, - ] - - -@pytest.mark.asyncio -async def test_execute_write_update_or_replace_requires_delete_row_permission(): - ds = Datasette( - memory=True, - default_deny=True, - config={ - "databases": { - "data": { - "permissions": { - "view-database": {"id": "writer"}, - "execute-write-sql": {"id": "writer"}, - }, - "tables": { - "users": { - "permissions": { - "insert-row": {"id": "writer"}, - "update-row": {"id": "writer"}, - "view-table": {"id": "writer"}, - } - } - }, - } - } - }, - ) - db = ds.add_memory_database("execute_write_update_or_replace", name="data") - await db.execute_write( - "create table users (id integer primary key, email text unique)" - ) - await db.execute_write( - "insert into users (id, email) values " - "(1, 'a@example.com'), (2, 'b@example.com')" - ) - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "writer"}, - json={ - "sql": "update or replace users set email = 'b@example.com' where id = 1" - }, + json={"sql": sql}, ) assert denied_response.status_code == 403 @@ -2262,74 +2219,71 @@ async def test_trusted_stored_write_query_skips_vacuum_filtering(): assert response.json()["ok"] is True +@pytest.mark.parametrize( + ( + "database_name", + "setup_sqls", + "write_sql", + "expected_error", + "verification_sql", + "expected_count", + ), + ( + ( + "execute_write_virtual_table_control", + ( + "create virtual table docs using fts5(title, body, content='')", + "insert into docs(rowid, title, body) values (1, 'hello', 'world')", + ), + "insert into docs(docs) values('delete-all')", + "Writes to virtual tables are not allowed in user-supplied SQL", + "select count(*) from docs where docs match 'hello'", + 1, + ), + ( + "execute_write_virtual_table_insert", + ("create virtual table docs using fts5(title, body)",), + "insert into docs(rowid, title, body) values (1, 'a', 'b')", + "Writes to virtual tables are not allowed in user-supplied SQL", + "select count(*) from docs", + 0, + ), + ( + "execute_write_shadow_table_insert", + ("create virtual table docs using fts5(title, body)",), + "insert into docs_config(k, v) values ('x', 1)", + "Writes to shadow tables are not allowed in user-supplied SQL", + "select count(*) from docs_config", + 1, + ), + ), + ids=("control-insert", "virtual-table", "shadow-table"), +) @pytest.mark.asyncio -async def test_execute_write_rejects_virtual_table_control_insert(): +async def test_execute_write_rejects_virtual_and_shadow_table_writes( + database_name, + setup_sqls, + write_sql, + expected_error, + verification_sql, + expected_count, +): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True - db = ds.add_memory_database("execute_write_virtual_table_control", name="data") - await db.execute_write(""" - create virtual table docs using fts5(title, body, content='') - """) - await db.execute_write(""" - insert into docs(rowid, title, body) values (1, 'hello', 'world') - """) + db = ds.add_memory_database(database_name, name="data") + for setup_sql in setup_sqls: + await db.execute_write(setup_sql) await ds.invoke_startup() denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "root"}, - json={"sql": "insert into docs(docs) values('delete-all')"}, + json={"sql": write_sql}, ) assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to virtual tables are not allowed in user-supplied SQL" - ] - assert ( - await db.execute("select count(*) from docs where docs match 'hello'") - ).first()[0] == 1 - - -@pytest.mark.asyncio -async def test_execute_write_rejects_regular_virtual_table_insert(): - ds = Datasette(memory=True, default_deny=True) - ds.root_enabled = True - db = ds.add_memory_database("execute_write_virtual_table_insert", name="data") - await db.execute_write("create virtual table docs using fts5(title, body)") - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "root"}, - json={"sql": "insert into docs(rowid, title, body) values (1, 'a', 'b')"}, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to virtual tables are not allowed in user-supplied SQL" - ] - assert (await db.execute("select count(*) from docs")).first()[0] == 0 - - -@pytest.mark.asyncio -async def test_execute_write_rejects_shadow_table_insert(): - ds = Datasette(memory=True, default_deny=True) - ds.root_enabled = True - db = ds.add_memory_database("execute_write_shadow_table_insert", name="data") - await db.execute_write("create virtual table docs using fts5(title, body)") - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "root"}, - json={"sql": "insert into docs_config(k, v) values ('x', 1)"}, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to shadow tables are not allowed in user-supplied SQL" - ] - assert (await db.execute("select count(*) from docs_config")).first()[0] == 1 + assert denied_response.json()["errors"] == [expected_error] + assert (await db.execute(verification_sql)).first()[0] == expected_count @pytest.mark.asyncio @@ -2482,8 +2436,69 @@ async def test_execute_write_create_table_uses_create_table_permission(): assert not await db.table_exists("should_not_exist") +@pytest.mark.parametrize( + ( + "database_name", + "allowed_actor", + "allowed_sql", + "denied_sql", + "expected_error", + "setup_sqls", + "expected_state", + ), + ( + ( + "execute_write_alter_table", + "alterer", + "alter table dogs add column age integer", + "alter table cats add column age integer", + "Permission denied: need alter-table on data/cats", + (), + "alter-table", + ), + ( + "execute_write_create_index", + "alterer", + "create index idx_dogs_name on dogs(name)", + "create index idx_cats_name on cats(name)", + "Permission denied: need alter-table on data/cats", + (), + "create-index", + ), + ( + "execute_write_drop_index", + "alterer", + "drop index idx_dogs_name", + "drop index idx_cats_name", + "Permission denied: need alter-table on data/cats", + ( + "create index idx_dogs_name on dogs(name)", + "create index idx_cats_name on cats(name)", + ), + "drop-index", + ), + ( + "execute_write_drop_table", + "dropper", + "drop table dogs", + "drop table cats", + "Permission denied: need drop-table on data/cats", + (), + "drop-table", + ), + ), + ids=("alter-table", "create-index", "drop-index", "drop-table"), +) @pytest.mark.asyncio -async def test_execute_write_alter_and_drop_table_use_schema_permissions(): +async def test_execute_write_schema_operations_use_schema_permissions( + database_name, + allowed_actor, + allowed_sql, + denied_sql, + expected_error, + setup_sqls, + expected_state, +): ds = Datasette( memory=True, default_deny=True, @@ -2513,73 +2528,53 @@ async def test_execute_write_alter_and_drop_table_use_schema_permissions(): }, }, ) - db = ds.add_memory_database("execute_write_alter_drop_table", name="data") + db = ds.add_memory_database(database_name, name="data") await db.execute_write("create table dogs (id integer primary key, name text)") await db.execute_write("create table cats (id integer primary key, name text)") + for setup_sql in setup_sqls: + await db.execute_write(setup_sql) await ds.invoke_startup() - alter_allowed_response = await ds.client.post( + async def index_exists(index_name): + row = ( + await db.execute( + "select 1 from sqlite_master where type = 'index' and name = ?", + [index_name], + ) + ).first() + return row is not None + + allowed_response = await ds.client.post( "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "alter table dogs add column age integer"}, + actor={"id": allowed_actor}, + json={"sql": allowed_sql}, ) - alter_row_permission_response = await ds.client.post( + denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "row-writer"}, - json={"sql": "alter table cats add column age integer"}, + json={"sql": denied_sql}, ) - assert alter_allowed_response.status_code == 200 - assert "age" in [column.name for column in await db.table_column_details("dogs")] - assert alter_row_permission_response.status_code == 403 - assert alter_row_permission_response.json()["errors"] == [ - "Permission denied: need alter-table on data/cats" - ] - assert "age" not in [ - column.name for column in await db.table_column_details("cats") - ] + assert allowed_response.status_code == 200 + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [expected_error] - create_index_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "create index idx_dogs_name on dogs(name)"}, - ) - create_index_row_permission_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "row-writer"}, - json={"sql": "create index idx_cats_name on cats(name)"}, - ) - drop_index_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "drop index idx_dogs_name"}, - ) - - assert create_index_allowed_response.status_code == 200 - assert create_index_row_permission_response.status_code == 403 - assert create_index_row_permission_response.json()["errors"] == [ - "Permission denied: need alter-table on data/cats" - ] - assert drop_index_allowed_response.status_code == 200 - - drop_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "dropper"}, - json={"sql": "drop table dogs"}, - ) - drop_row_permission_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "row-writer"}, - json={"sql": "drop table cats"}, - ) - - assert drop_allowed_response.status_code == 200 - assert not await db.table_exists("dogs") - assert drop_row_permission_response.status_code == 403 - assert drop_row_permission_response.json()["errors"] == [ - "Permission denied: need drop-table on data/cats" - ] - assert await db.table_exists("cats") + if expected_state == "alter-table": + assert "age" in [ + column.name for column in await db.table_column_details("dogs") + ] + assert "age" not in [ + column.name for column in await db.table_column_details("cats") + ] + elif expected_state == "create-index": + assert await index_exists("idx_dogs_name") + assert not await index_exists("idx_cats_name") + elif expected_state == "drop-index": + assert not await index_exists("idx_dogs_name") + assert await index_exists("idx_cats_name") + elif expected_state == "drop-table": + assert not await db.table_exists("dogs") + assert await db.table_exists("cats") @pytest.mark.asyncio @@ -2644,8 +2639,9 @@ async def test_execute_write_post_rejects_read_only_sql(): ] +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) @pytest.mark.asyncio -async def test_query_owner_gets_update_delete_and_writable_view_defaults(): +async def test_query_owner_gets_update_delete_and_writable_view_defaults(action): ds = Datasette(memory=True, default_deny=True) ds.add_memory_database("query_owner_defaults", name="data") await ds.invoke_startup() @@ -2658,21 +2654,35 @@ async def test_query_owner_gets_update_delete_and_writable_view_defaults(): owner_id="alice", ) - for action in ("view-query", "update-query", "delete-query"): - assert await ds.allowed( - action=action, - resource=QueryResource("data", "insert_dog"), - actor={"id": "alice"}, - ) - assert not await ds.allowed( - action=action, - resource=QueryResource("data", "insert_dog"), - actor={"id": "bob"}, - ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "bob"}, + ) +@pytest.mark.parametrize( + "action, path_suffix, request_json, expected_public_title", + ( + ( + "update-query", + "-/update", + {"update": {"title": "Bob can edit public queries"}}, + "Bob can edit public queries", + ), + ("delete-query", "-/delete", {}, None), + ), + ids=("update-query", "delete-query"), +) @pytest.mark.asyncio -async def test_private_query_restricts_broad_update_delete_permissions(): +async def test_private_query_restricts_broad_update_delete_permissions( + action, path_suffix, request_json, expected_public_title +): ds = Datasette( memory=True, default_deny=True, @@ -2706,50 +2716,41 @@ async def test_private_query_restricts_broad_update_delete_permissions(): owner_id="alice", ) - for action in ("update-query", "delete-query"): - assert await ds.allowed( - action=action, - resource=QueryResource("data", "alice_private"), - actor={"id": "alice"}, - ) - assert not await ds.allowed( - action=action, - resource=QueryResource("data", "alice_private"), - actor={"id": "bob"}, - ) - assert await ds.allowed( - action=action, - resource=QueryResource("data", "alice_public"), - actor={"id": "bob"}, - ) - - private_update_response = await ds.client.post( - "/data/alice_private/-/update", - actor={"id": "bob"}, - json={"update": {"title": "Nope"}}, + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, ) - private_delete_response = await ds.client.post( - "/data/alice_private/-/delete", + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), actor={"id": "bob"}, - json={}, ) - public_update_response = await ds.client.post( - "/data/alice_public/-/update", + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), actor={"id": "bob"}, - json={"update": {"title": "Bob can edit public queries"}}, - ) - public_delete_response = await ds.client.post( - "/data/alice_public/-/delete", - actor={"id": "bob"}, - json={}, ) - assert private_update_response.status_code == 403 - assert private_delete_response.status_code == 403 - assert public_update_response.status_code == 200 - assert public_delete_response.status_code == 200 + private_response = await ds.client.post( + "/data/alice_private/{}".format(path_suffix), + actor={"id": "bob"}, + json=request_json, + ) + public_response = await ds.client.post( + "/data/alice_public/{}".format(path_suffix), + actor={"id": "bob"}, + json=request_json, + ) + + assert private_response.status_code == 403 + assert public_response.status_code == 200 assert await ds.get_query("data", "alice_private") is not None - assert await ds.get_query("data", "alice_public") is None + public_query = await ds.get_query("data", "alice_public") + if expected_public_title is None: + assert public_query is None + else: + assert public_query.title == expected_public_title @pytest.mark.asyncio From b6e9b189905f6a03136e5998fdf39e1944a1e2a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:37:48 -0700 Subject: [PATCH 1814/1866] datasette.yml can no longer set a query to private Private means it has an owner, and the config does not let you say who the owner is - plus configured queries should not be possible to edit or delete in the UI so having an owner makes even less sense. You can still make configured queries visible to specific people using regular view-query permissions. --- datasette/stored_queries.py | 1 - tests/test_queries.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index b6ac49b8..a6123daa 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -109,7 +109,6 @@ async def save_queries_from_config(datasette: Any) -> None: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), is_trusted=bool(query_config.get("is_trusted", True)), source="config", on_success_message=query_config.get("on_success_message"), diff --git a/tests/test_queries.py b/tests/test_queries.py index 216cb211..2aa5142b 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -191,6 +191,8 @@ async def test_config_queries_imported_to_internal_table(): "title": "Configured query", "description_html": "

    Configured HTML

    ", "params": ["name"], + # Configured queries are always public; this is ignored. + "is_private": True, "on_success_message_sql": "select 'Hello ' || :name", } } From 74324cb8492be8aa8597e58fb6f690158128e6fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:46:27 -0700 Subject: [PATCH 1815/1866] Improved docs for user-facing SQL query pages - /database-name/-/execute-write - /-/queries --- docs/authentication.rst | 4 ++-- docs/pages.rst | 27 +++++++++++++++++++++++++++ docs/sql_queries.rst | 2 ++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a0891900..5d831da0 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1413,7 +1413,7 @@ Actor is allowed to drop a database table. execute-sql ----------- -Actor is allowed to run arbitrary read-only SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 +Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page `, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) @@ -1425,7 +1425,7 @@ See also :ref:`the default_allow_sql setting `. execute-write-sql ----------------- -Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. +Actor is allowed to run arbitrary writable SQL queries against a specific database using the :ref:`write SQL queries page `, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/pages.rst b/docs/pages.rst index e57c15e6..a8ff7c37 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -62,6 +62,11 @@ The following tables are hidden by default: Queries ======= +.. _pages_custom_sql_queries: + +Custom SQL queries +------------------ + The ``/database-name/-/query`` page can be used to execute an arbitrary SQL query against that database, if the :ref:`actions_execute_sql` permission is enabled. This query is passed as the ``?sql=`` query string parameter. This means you can link directly to a query by constructing the following URL: @@ -72,6 +77,28 @@ Each configured :ref:`stored query ` has its own page, at ``/dat In both cases adding a ``.json`` extension to the URL will return the results as JSON. +.. _pages_execute_write: + +Write SQL queries +----------------- + +The ``/database-name/-/execute-write`` page can be used to execute SQL statements that write to a mutable database, if the :ref:`actions_execute_write_sql` permission is enabled. + +This page extracts named parameters from the SQL, shows the tables that will be affected and lists the permissions required before the query can be executed. It also includes templates for common ``INSERT``, ``UPDATE`` and ``DELETE`` statements. + +Datasette checks additional permissions based on the operations in the SQL. Row changes require the relevant table-level permissions such as :ref:`actions_insert_row`, :ref:`actions_update_row` and :ref:`actions_delete_row`; reads from source tables require :ref:`actions_view_table`; and schema changes require permissions such as :ref:`actions_create_table`, :ref:`actions_alter_table` or :ref:`actions_drop_table`. + +Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmatically. + +.. _pages_stored_query_browser: + +Stored query browsers +--------------------- + +The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. + +These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON. + .. _TableView: Table diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index d427ea2b..c0ba67f0 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -7,6 +7,8 @@ Datasette treats SQLite database files as read-only and immutable. This means it The easiest way to execute custom SQL against Datasette is through the web UI. The database index page includes a SQL editor that lets you run any SELECT query you like. You can also construct queries using the filter interface on the tables page, then click "View and edit SQL" to open that query in the custom SQL editor. +For mutable databases, actors with the appropriate permissions can use the :ref:`write SQL page ` to execute SQL statements that insert, update or delete rows. + Note that this interface is only available if the :ref:`actions_execute_sql` permission is allowed. See :ref:`authentication_permissions_execute_sql`. Any Datasette SQL query is reflected in the URL of the page, allowing you to bookmark them, share them with others and navigate through previous queries using your browser back button. From 6a998610eef6e69d439a654dd31087023d285452 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:52:51 -0700 Subject: [PATCH 1816/1866] datasette inspect now counts 10,000+ tables correctly (#2752) Closes #2712 Refs https://github.com/simonw/datasette/pull/2721#issuecomment-4568966383 --- datasette/cli.py | 7 ++++--- docs/changelog.rst | 1 + tests/test_cli.py | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 93aa22ef..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( SQLITE_LIMIT_ATTACHED, pm, ) +from .inspect import inspect_tables from .utils import ( LoadExtension, StartupError, @@ -154,14 +155,14 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - counts = await database.table_counts(limit=3600 * 1000) + tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) data[name] = { "hash": database.hash, "size": database.size, "file": database.path, "tables": { - table_name: {"count": table_count} - for table_name, table_count in counts.items() + table_name: {"count": table["count"]} + for table_name, table in tables.items() }, } return data diff --git a/docs/changelog.rst b/docs/changelog.rst index a4be98b1..3882cc12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ Bug fixes ~~~~~~~~~ - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) +- The ``datasette inspect`` command now correctly records row counts for tables with more than 10,000 rows. (:issue:`2712`) .. _v1_0_a30: diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d3a2b28..f86d6909 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -35,12 +35,28 @@ def test_inspect_cli(app_client): assert expected_count == database["tables"][table_name]["count"] +def test_inspect_cli_counts_all_rows(tmp_path): + db_path = tmp_path / "big.db" + conn = sqlite3.connect(db_path) + with conn: + conn.execute("create table t (id integer primary key)") + conn.executemany("insert into t (id) values (?)", ((i,) for i in range(10002))) + conn.close() + + runner = CliRunner() + result = runner.invoke(cli, ["inspect", str(db_path)]) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + + assert data["big"]["tables"]["t"]["count"] == 10002 + + def test_inspect_cli_writes_to_file(app_client): runner = CliRunner() result = runner.invoke( cli, ["inspect", "fixtures.db", "--inspect-file", "foo.json"] ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output with open("foo.json") as fp: data = json.load(fp) assert ["fixtures"] == list(data.keys()) From e5b6166fa35558920342e74f5ec13078957e87bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 16:19:39 -0700 Subject: [PATCH 1817/1866] Nicer UI around Execute Write SQL denied Refs https://github.com/simonw/datasette/issues/2753#issuecomment-4569117665 --- datasette/templates/execute_write.html | 82 ++++++++++++++++++++------ datasette/views/execute_write.py | 17 +++--- datasette/views/query_helpers.py | 20 +++++-- tests/test_queries.py | 75 ++++++++++++++++++++++- 4 files changed, 160 insertions(+), 34 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 6b626f8d..ee251111 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -40,6 +40,26 @@ border-radius: 0.25rem; min-width: 13rem; } +.execute-write-submit-row { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 0.45rem 0.75rem; +} +.execute-write-submit-row [hidden] { + display: none; +} +form.sql.core input[data-execute-write-submit]:disabled { + background: #d0d7de; + border-color: #b6c0cc; + color: #5f6975; + cursor: not-allowed; + opacity: 1; +} +.execute-write-disabled-reason { + color: #4f5b6d; + font-size: 0.85rem; +} {% include "_execute_write_analysis_styles.html" %} {% include "_sql_parameter_styles.html" %} @@ -119,9 +139,10 @@ {% endif %}
    -

    - - {% if save_query_base_url %}Save this query{% endif %} +

    + + {{ execute_disabled_reason or "" }} + {% if save_query_url %}Save this query{% endif %}

    @@ -143,25 +164,55 @@ window.addEventListener("DOMContentLoaded", () => { const submitButton = form ? form.querySelector("[data-execute-write-submit]") : null; - const saveQueryLink = form + const submitDisabledReason = form + ? form.querySelector("[data-execute-write-disabled-reason]") + : null; + const submitRow = form + ? form.querySelector(".execute-write-submit-row") + : null; + let saveQueryLink = form ? form.querySelector("[data-save-query-link]") : null; + function updateSubmitState(data) { + if (submitButton) { + submitButton.disabled = data.execute_disabled; + } + if (!submitDisabledReason) { + return; + } + const reason = data.execute_disabled_reason || ""; + submitDisabledReason.textContent = reason; + submitDisabledReason.hidden = !reason; + } + function updateSaveQueryLink(data) { - if (!saveQueryLink) { + if (!submitRow || !submitRow.dataset.saveQueryBaseUrl) { return; } const sql = window.editor ? window.editor.state.doc.toString() : executeWriteSqlInput.value; if (!sql.trim() || !data.ok || data.execute_disabled) { - saveQueryLink.hidden = true; + if (saveQueryLink) { + saveQueryLink.remove(); + saveQueryLink = null; + } return; } - const url = new URL(saveQueryLink.dataset.saveQueryBaseUrl, window.location.href); + if (!saveQueryLink) { + saveQueryLink = document.createElement("a"); + saveQueryLink.className = "save-query"; + saveQueryLink.setAttribute("data-save-query-link", ""); + saveQueryLink.textContent = "Save this query"; + submitRow.appendChild(saveQueryLink); + } + const url = new URL( + submitRow.dataset.saveQueryBaseUrl, + window.location.href + ); url.searchParams.set("sql", sql); saveQueryLink.href = url.pathname + url.search + url.hash; - saveQueryLink.hidden = false; } window.datasetteSqlParameters.setupSqlParameterRefresh({ @@ -170,9 +221,7 @@ window.addEventListener("DOMContentLoaded", () => { allowExpand: true, onData(data) { window.datasetteSqlAnalysis.renderAnalysis(analysisSection, data); - if (submitButton) { - submitButton.disabled = data.execute_disabled; - } + updateSubmitState(data); updateSaveQueryLink(data); }, onError(error) { @@ -180,12 +229,11 @@ window.addEventListener("DOMContentLoaded", () => { analysis_error: error.message, analysis_rows: [], }); - if (submitButton) { - submitButton.disabled = true; - } - if (saveQueryLink) { - saveQueryLink.hidden = true; - } + updateSubmitState({ + execute_disabled: true, + execute_disabled_reason: error.message, + }); + updateSaveQueryLink({ ok: false, execute_disabled: true }); }, }); }); diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 57c4d78e..7b693978 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -14,6 +14,7 @@ from .query_helpers import ( _coerce_execute_write_payload, _derived_query_parameters, _execute_write_analysis_data, + _execute_write_disabled_reason, _inserted_row_url, _json_or_form_payload, _prepare_execute_write, @@ -80,13 +81,12 @@ class ExecuteWriteView(BaseView): ) save_query_base_url = None save_query_url = None + execute_disabled_reason = _execute_write_disabled_reason( + sql, analysis_error, analysis_rows + ) if allow_save_query: save_query_base_url = self.ds.urls.database(db.name) + "/-/queries/store" - if ( - sql - and analysis_error is None - and not any(row["allowed"] is False for row in analysis_rows) - ): + if not execute_disabled_reason: save_query_url = save_query_base_url + "?" + urlencode({"sql": sql}) response = await self.render( @@ -103,11 +103,8 @@ class ExecuteWriteView(BaseView): "execution_message": execution_message, "execution_links": execution_links, "execution_ok": execution_ok, - "execute_disabled": bool( - (not sql) - or analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), + "execute_disabled": bool(execute_disabled_reason), + "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, "write_template_tables": write_template_tables, "save_query_url": save_query_url, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 712832e8..f30a30bc 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -268,6 +268,16 @@ async def _analysis_rows_with_permissions( return rows +def _execute_write_disabled_reason(sql, analysis_error, analysis_rows): + if not (sql and sql.strip()): + return "Enter writable SQL before executing." + if analysis_error: + return analysis_error + if any(row.get("allowed") is False for row in analysis_rows): + return "You do not have permission for every operation listed above." + return None + + def _coerce_execute_write_payload(data, is_json): if not isinstance(data, dict): raise QueryValidationError("JSON must be a dictionary") @@ -358,16 +368,16 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): ) except (QueryValidationError, sqlite3.DatabaseError) as ex: analysis_error = getattr(ex, "message", str(ex)) + execute_disabled_reason = _execute_write_disabled_reason( + sql, analysis_error, analysis_rows + ) return { "ok": analysis_error is None, "parameters": parameter_names, "analysis_error": analysis_error, "analysis_rows": analysis_rows, - "execute_disabled": bool( - (not sql) - or analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), + "execute_disabled": bool(execute_disabled_reason), + "execute_disabled_reason": execute_disabled_reason, } diff --git a/tests/test_queries.py b/tests/test_queries.py index 2aa5142b..87ecacde 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1374,6 +1374,10 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'addEventListener("paste"' in response.text assert "setupSqlParameterRefresh" in response.text assert "datasetteSqlAnalysis.renderAnalysis" in response.text + assert "input[data-execute-write-submit]:disabled" in response.text + assert ( + 'data-execute-write-disabled-reason aria-live="polite" hidden' in response.text + ) assert '' in response.text assert '' in response.text assert "" in response.text @@ -1390,7 +1394,9 @@ async def test_execute_write_get_prepopulates_without_executing(): ) assert '' in empty_response.text assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text - assert "hidden>Save this query" in empty_response.text + assert "Enter writable SQL before executing." in empty_response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in empty_response.text + assert 'Save this query" in read_only_response.text + assert ( + '' + ) in read_only_response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in read_only_response.text + assert '' + ) in response.text + assert ( + '' + "You do not have permission for every operation listed above." + ) in response.text + assert 'no' in response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in response.text + assert ' Date: Thu, 28 May 2026 16:20:28 -0700 Subject: [PATCH 1818/1866] //-/query.json and changelog docs --- docs/changelog.rst | 3 ++- docs/json_api.rst | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3882cc12..3501aa60 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,12 +17,13 @@ Stored queries - Users with :ref:`store-query ` and :ref:`execute-sql ` permission can create stored queries from the SQL query page or the new ``GET //-/queries/store`` form. (:issue:`2735`) - The database page now shows a count and preview of stored queries, capped at five, and links to new paginated query browsers at ``/-/queries`` and ``//-/queries``. Those browsers support search. (:issue:`2735`) - Stored queries created by users default to private and untrusted. Private stored queries can only be viewed, updated or deleted by their owner, even if another actor has broad ``view-query``, ``update-query`` or ``delete-query`` permission. Untrusted stored queries execute using the permissions of the actor running them. See :ref:`stored_queries` and :ref:`trusted_stored_queries` for details. (:issue:`2735`) +- Configured queries from ``datasette.yaml`` are trusted by default, so they can execute with ``view-query`` permission alone. They can opt out of that behavior using ``is_trusted: false`` but cannot be made private; private queries are only available for user-created stored queries. (:issue:`2735`) - New ``store-query``, ``update-query`` and ``delete-query`` permissions, plus updated semantics for :ref:`view-query `. Trusted stored queries can still execute with ``view-query`` alone; untrusted read queries also require :ref:`execute-sql ` and untrusted writable queries require :ref:`execute-write-sql ` plus the relevant table-level write permissions. (:issue:`2735`) Write SQL UI ~~~~~~~~~~~~ -- New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`) +- New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted, includes starter templates for ``INSERT``, ``UPDATE`` and ``DELETE`` statements and links to a newly inserted row when a single-row insert succeeds. This is also available as a :ref:`JSON API `. (:issue:`2742`) - Added the new :ref:`execute-write-sql ` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row `, :ref:`update-row ` and :ref:`delete-row `, and writes to attached databases are rejected. (:issue:`2742`) - The write SQL analyzer now uses a deny-by-default model for unsupported operations. Reads from source tables require :ref:`view-table ` permission, schema changes require :ref:`create-table `, :ref:`alter-table ` or :ref:`drop-table ` as appropriate, and row mutation statements require the full ``insert-row``, ``update-row`` and ``delete-row`` permission set. SQL functions are allowed and are not separately permission-gated. (:issue:`2748`) - User-supplied write SQL now rejects ``VACUUM`` and writes to SQLite virtual tables or shadow tables. These restrictions also apply to untrusted stored write queries; trusted configured stored queries continue to skip these filters. (:issue:`2748`) diff --git a/docs/json_api.rst b/docs/json_api.rst index db19afc2..4bd76717 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -50,6 +50,25 @@ The ``"truncated"`` key lets you know if the query was truncated. This can happe For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. +.. _json_api_custom_sql: + +Executing custom SQL +-------------------- + +Actors with the :ref:`actions_execute_sql` permission can execute read-only SQL against a database using ``/-/query.json``: + +:: + + GET //-/query.json?sql=select+*+from+dogs + +Values for named SQL parameters can be provided as additional query string parameters: + +:: + + GET //-/query.json?sql=select+*+from+dogs+where+name=:name&name=Cleo + +The response uses the same default representation described above. + .. _json_api_shapes: Different shapes @@ -529,7 +548,7 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr } } -The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. +The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query JSON API ` instead. Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. Schema changes require ``create-table``, ``alter-table`` or ``drop-table`` permissions as appropriate. From 9e377e8b90b27ae21d3263d0bfe8d3808e2c6133 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 20:01:48 -0700 Subject: [PATCH 1819/1866] Only show valid SQL write templates Closes #2753 Demo: https://github.com/simonw/datasette/issues/2753#issuecomment-4570071413 --- datasette/templates/execute_write.html | 130 ++++------------- datasette/views/execute_write.py | 192 ++++++++++++++++++++++++- tests/test_queries.py | 117 ++++++++++++++- 3 files changed, 331 insertions(+), 108 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index ee251111..394261de 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -89,16 +89,18 @@ form.sql.core input[data-execute-write-submit]:disabled {

    - - - + {% for operation in write_template_operations %} + + {% endfor %}

    + {% else %} +

    You don't currently have permission to insert, edit or delete from any tables.

    {% endif %}

    @@ -242,119 +244,43 @@ window.addEventListener("DOMContentLoaded", () => { {% if write_template_tables %} {% endif %} diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 7b693978..cff20847 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -1,3 +1,4 @@ +import re from urllib.parse import urlencode from datasette.resources import DatabaseResource @@ -22,6 +23,187 @@ from .query_helpers import ( _wants_json, ) +WRITE_TEMPLATE_LABELS = { + "insert": "Insert row", + "update": "Update rows", + "delete": "Delete rows", +} +WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) + + +def _parameter_names(columns): + seen = set() + names = {} + for column in columns: + base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) + base = base.strip("_") or "value" + if base[0].isdigit(): + base = "p_{}".format(base) + name = base + index = 2 + while name in seen: + name = "{}_{}".format(base, index) + index += 1 + seen.add(name) + names[column] = name + return names + + +def _quote_identifier(identifier): + return '"{}"'.format(identifier.replace('"', '""')) + + +def _preferred_where_column(table, columns): + lower_table_id = "{}_id".format(table.lower()) + return ( + next((column for column in columns if column.lower() == "id"), None) + or next( + (column for column in columns if column.lower() == lower_table_id), None + ) + or columns[0] + ) + + +def _auto_incrementing_primary_key(columns): + primary_keys = [column for column in columns if column.is_pk] + if len(primary_keys) != 1: + return None + primary_key = primary_keys[0] + if primary_key.type and primary_key.type.lower() == "integer": + return primary_key.name + return None + + +def _insert_template_sql(table, columns): + column_names = [column.name for column in columns] + auto_pk = _auto_incrementing_primary_key(columns) + insert_columns = [column for column in column_names if column != auto_pk] + if not insert_columns: + return "insert into {}\ndefault values".format(_quote_identifier(table)) + names = _parameter_names(insert_columns) + return "\n".join( + ( + "insert into {} (".format(_quote_identifier(table)), + ",\n".join( + " {}".format(_quote_identifier(column)) for column in insert_columns + ), + ")", + "values (", + ",\n".join(" :{}".format(names[column]) for column in insert_columns), + ")", + ) + ) + + +def _update_template_sql(table, columns): + column_names = [column.name for column in columns] + names = _parameter_names(column_names) + where_column = _preferred_where_column(table, column_names) + set_columns = [column for column in column_names if column != where_column] + if not set_columns: + return "\n".join( + ( + "update {}".format(_quote_identifier(table)), + "set {} = :new_{}".format( + _quote_identifier(where_column), names[where_column] + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + return "\n".join( + ( + "update {}".format(_quote_identifier(table)), + "set " + + ",\n".join( + "{}{} = :{}".format( + " " if index else "", + _quote_identifier(column), + names[column], + ) + for index, column in enumerate(set_columns) + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + + +def _delete_template_sql(table, columns): + column_names = [column.name for column in columns] + names = _parameter_names(column_names) + where_column = _preferred_where_column(table, column_names) + return "\n".join( + ( + "delete from {}".format(_quote_identifier(table)), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + + +def _template_sqls_for_table(table, columns): + return { + "insert": _insert_template_sql(table, columns), + "update": _update_template_sql(table, columns), + "delete": _delete_template_sql(table, columns), + } + + +async def _template_sql_allowed(datasette, db, sql, actor): + params = {parameter: "" for parameter in _derived_query_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError: + return False + if not _analysis_is_write(analysis): + return False + analysis_rows = await _analysis_rows_with_permissions(datasette, analysis, actor) + return _execute_write_disabled_reason(sql, None, analysis_rows) is None + + +async def _write_template_tables( + datasette, db, table_columns, hidden_table_names, actor +): + write_template_tables = {} + for table in table_columns: + if table in hidden_table_names or not table_columns[table]: + continue + column_details = [ + column + for column in await db.table_column_details(table) + if not column.hidden + ] + if not column_details: + continue + templates = {} + for operation, sql in _template_sqls_for_table(table, column_details).items(): + if await _template_sql_allowed(datasette, db, sql, actor): + templates[operation] = sql + if templates: + write_template_tables[table] = { + "templates": templates, + } + return write_template_tables + + +def _write_template_operations(write_template_tables): + operations = [] + for operation in WRITE_TEMPLATE_OPERATIONS: + if any( + operation in table["templates"] for table in write_template_tables.values() + ): + operations.append( + { + "name": operation, + "label": WRITE_TEMPLATE_LABELS[operation], + } + ) + return operations + class ExecuteWriteView(BaseView): name = "execute-write" @@ -47,11 +229,10 @@ class ExecuteWriteView(BaseView): analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) hidden_table_names = set(await db.hidden_table_names()) - write_template_tables = { - table: columns - for table, columns in table_columns.items() - if columns and table not in hidden_table_names - } + write_template_tables = await _write_template_tables( + self.ds, db, table_columns, hidden_table_names, request.actor + ) + write_template_operations = _write_template_operations(write_template_tables) if sql and analysis_error is None: try: parameter_names = _derived_query_parameters(sql) @@ -107,6 +288,7 @@ class ExecuteWriteView(BaseView): "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, "write_template_tables": write_template_tables, + "write_template_operations": write_template_operations, "save_query_url": save_query_url, "save_query_base_url": save_query_base_url, }, diff --git a/tests/test_queries.py b/tests/test_queries.py index 87ecacde..89167a1d 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,4 +1,6 @@ import json +import re +from html import unescape import pytest @@ -8,6 +10,19 @@ from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden +def _template_option_attributes(html, table): + match = re.search(r'' in response.text + assert '
    POST - +
    diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 7770f7d4..a46478a7 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -11,7 +11,7 @@
    - + - - + + - + - - + + - + - - + +
    Required permissioninsert - rowid ▼ + rowid ▼ - attraction_id + attraction_id - characteristic_id + characteristic_id
    11 1The Mystery Spot 1Paranormal 2The Mystery Spot 1Paranormal 2
    22 2Winchester Mystery House 2Paranormal 2Winchester Mystery House 2Paranormal 2
    33 3Bigfoot Discovery Museum 4Paranormal 2Bigfoot Discovery Museum 4Paranormal 2

    Advanced export

    JSON shape: - default, - array, - newline-delimited + default, + array, + newline-delimited

    - +

    CSV options: @@ -445,7 +445,7 @@

    .bd for /database/table/row

    roadside_attractions: 2

    -

    This data as json

    +

    This data as json

    @@ -479,7 +479,7 @@

    Links from other tables

    • - + 1 row from attraction_id in roadside_attraction_characteristics
      • diff --git a/datasette/views/database.py b/datasette/views/database.py index 3e3b05e3..d6c88962 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -60,9 +60,11 @@ class DatabaseView(View): sql = (request.args.get("sql") or "").strip() if sql: - redirect_url = "/" + request.url_vars.get("database") + "/-/query" + redirect_url = datasette.urls.database(database) + "/-/query" if request.url_vars.get("format"): - redirect_url += "." + request.url_vars.get("format") + redirect_url = path_with_format( + path=redirect_url, format=request.url_vars.get("format") + ) redirect_url += "?" + request.query_string response = Response.redirect(redirect_url) if datasette.cors: diff --git a/datasette/views/special.py b/datasette/views/special.py index 6c82983c..75c54c3c 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -892,14 +892,15 @@ class ApiExplorerView(BaseView): raise Forbidden("You do not have permission to view this instance") def api_path(link): - return "/-/api#{}".format( + return "{}#{}".format( + self.ds.urls.path("/-/api"), urllib.parse.urlencode( { key: json.dumps(value, indent=2) if key == "json" else value for key, value in link.items() if key in ("path", "method", "json") } - ) + ), ) return await self.render( diff --git a/tests/test_html.py b/tests/test_html.py index 96ee9c0c..bb7f612e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -878,6 +878,8 @@ def test_debug_context_includes_extra_template_vars(): "/fixtures/facetable", "/fixtures/facetable?_facet=state", "/fixtures/-/query?sql=select+1", + "/-/api", + "/-/patterns", ], ) @pytest.mark.parametrize("use_prefix", (True, False)) @@ -932,7 +934,9 @@ def test_base_url_config(app_client_base_url_prefix, path, use_prefix): ): # If this has been made absolute it may start http://localhost/ if href.startswith("http://localhost/"): - href = href[len("http://localost/") :] + href = href[len("http://localhost") :] + elif href.startswith(("http://", "https://")): + continue assert href.startswith("/prefix/"), json.dumps( { "path": path, @@ -966,6 +970,25 @@ def test_base_url_affects_filter_redirects(app_client_base_url_prefix): ) +def test_base_url_affects_database_sql_redirect(app_client_base_url_prefix): + response = app_client_base_url_prefix.get( + "/prefix/fixtures?sql=select+1", follow_redirects=False + ) + assert response.status_code == 302 + assert response.headers["location"] == "/prefix/fixtures/-/query?sql=select+1" + + +def test_base_url_affects_permanent_redirects(): + with make_app_client(memory=True, settings={"base_url": "/prefix/"}) as client: + response = client.get("/prefix/-", follow_redirects=False) + assert response.status_code == 301 + assert response.headers["location"] == "/prefix/-/" + + response2 = client.get("/prefix/:memory:", follow_redirects=False) + assert response2.status_code == 301 + assert response2.headers["location"] == "/prefix/_memory" + + def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): html = app_client_base_url_prefix.get("/").text assert '' in html From b1f3e4368c81490c1468b1c641e02fa15771b013 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2026 16:15:34 -0700 Subject: [PATCH 1825/1866] Fixes for SQL write with RETURNING (#2763) * Fix for execute write returning, closes #2762 * Fix stored write returning rowcount message * Add configurable execute_write returning limit * Return rows/truncated from execute query if it used RETURNING * INSERT ... RETURNING shows rows in /-/execute-write * Skip RETURNING tests if SQLite version does not support it Screenshot: https://github.com/simonw/datasette/issues/2762#issuecomment-4588111545 --- datasette/database.py | 57 ++++++- datasette/templates/_query_results.html | 20 +++ datasette/templates/execute_write.html | 11 ++ datasette/templates/query.html | 22 +-- datasette/utils/sqlite.py | 16 ++ datasette/views/database.py | 10 +- datasette/views/execute_write.py | 59 +++++-- docs/internals.rst | 29 +++- docs/json_api.rst | 42 ++++- tests/test_internals_database.py | 181 ++++++++++++++++++++- tests/test_queries.py | 201 ++++++++++++++++++++++++ tests/test_utils.py | 21 ++- 12 files changed, 622 insertions(+), 47 deletions(-) create mode 100644 datasette/templates/_query_results.html diff --git a/datasette/database.py b/datasette/database.py index 10417670..0a32442c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -31,6 +31,8 @@ from .inspect import inspect_hash connections = threading.local() +EXECUTE_WRITE_RETURNING_LIMIT = 10 + AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) @@ -236,11 +238,24 @@ class Database: except OSError: pass - async def execute_write(self, sql, params=None, block=True, request=None): + async def execute_write( + self, + sql, + params=None, + block=True, + request=None, + return_all=False, + returning_limit=EXECUTE_WRITE_RETURNING_LIMIT, + ): self._check_not_closed() + if returning_limit < 0: + raise ValueError("returning_limit must be >= 0") def _inner(conn): - return conn.execute(sql, params or []) + cursor = conn.execute(sql, params or []) + return ExecuteWriteResult.from_cursor( + cursor, return_all=return_all, returning_limit=returning_limit + ) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block, request=request) @@ -877,6 +892,44 @@ class MultipleValues(Exception): pass +class ExecuteWriteResult: + def __init__(self, rowcount, lastrowid, description, rows, truncated): + self.rowcount = rowcount + self.lastrowid = lastrowid + self.description = description + self.truncated = truncated + self._rows = rows + + @classmethod + def from_cursor( + cls, cursor, return_all=False, returning_limit=EXECUTE_WRITE_RETURNING_LIMIT + ): + rows = [] + truncated = False + description = cursor.description + lastrowid = cursor.lastrowid + try: + if description is not None: + if return_all: + rows = cursor.fetchall() + else: + rows = cursor.fetchmany(returning_limit + 1) + if len(rows) > returning_limit: + rows = rows[:returning_limit] + truncated = True + rowcount = cursor.rowcount + finally: + cursor.close() + if description is not None and not return_all and truncated: + rowcount = -1 + return cls(rowcount, lastrowid, description, rows, truncated) + + def fetchall(self): + rows = self._rows + self._rows = [] + return rows + + class Results: def __init__(self, rows, truncated, description): self.rows = rows diff --git a/datasette/templates/_query_results.html b/datasette/templates/_query_results.html new file mode 100644 index 00000000..5e1e2f72 --- /dev/null +++ b/datasette/templates/_query_results.html @@ -0,0 +1,20 @@ +{% if display_rows %} +
    + + + {% for column in columns %}{% endfor %} + + + + {% for row in display_rows %} + + {% for column, td in zip(columns, row) %} + + {% endfor %} + + {% endfor %} + +
    {{ column }}
    {{ td }}
    +{% elif show_zero_results %} +

    0 results

    +{% endif %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 394261de..a93de3a6 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -81,6 +81,17 @@ form.sql.core input[data-execute-write-submit]:disabled {

    {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

    {% endif %} +{% if execute_write_returns_rows %} +

    Returned rows

    + {% if execute_write_truncated %} +

    Only the first {{ "{:,}".format(execute_write_display_rows|length) }} returned rows are shown.

    + {% endif %} + {% set columns = execute_write_columns %} + {% set display_rows = execute_write_display_rows %} + {% set show_zero_results = true %} + {% include "_query_results.html" %} +{% endif %} + {% if write_template_tables %}
    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 168a636b..8dd1037f 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -73,27 +73,9 @@ {% if display_rows %}

    This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}, CSV

    -
    - - - {% for column in columns %}{% endfor %} - - - - {% for row in display_rows %} - - {% for column, td in zip(columns, row) %} - - {% endfor %} - - {% endfor %} - -
    {{ column }}
    {{ td }}
    -{% else %} - {% if not stored_query_write and not error %} -

    0 results

    - {% endif %} {% endif %} +{% set show_zero_results = not stored_query_write and not error %} +{% include "_query_results.html" %} {% include "_codemirror_foot.html" %} {% include "_sql_parameter_scripts.html" %} diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 5a7c6c38..4743ae4c 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -13,6 +13,7 @@ if hasattr(sqlite3, "enable_callback_tracebacks"): sqlite3.enable_callback_tracebacks(True) _cached_sqlite_version = None +_cached_supports_returning = None SQLiteTableType = Literal["table", "view", "virtual", "shadow"] _VIRTUAL_TABLE_MODULE_RE = re.compile( r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", @@ -59,6 +60,21 @@ def supports_generated_columns(): return sqlite_version() >= (3, 31, 0) +def supports_returning(): + global _cached_supports_returning + if _cached_supports_returning is None: + conn = sqlite3.connect(":memory:") + try: + conn.execute("create table t (id integer primary key)") + conn.execute("insert into t default values returning id").fetchone() + _cached_supports_returning = True + except sqlite3.DatabaseError: + _cached_supports_returning = False + finally: + conn.close() + return _cached_supports_returning + + def sqlite_table_type( conn, table: str, diff --git a/datasette/views/database.py b/datasette/views/database.py index d6c88962..a1647ca9 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -528,12 +528,14 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = ( - stored_query.on_success_message - or "Query executed, {} row{} affected".format( + if stored_query.on_success_message: + message = stored_query.on_success_message + elif cursor.rowcount == -1: + message = "Query executed" + else: + message = "Query executed, {} row{} affected".format( cursor.rowcount, "" if cursor.rowcount == 1 else "s" ) - ) redirect_url = stored_query.on_success_redirect ok = True diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index cff20847..c5d55b80 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -6,6 +6,7 @@ from datasette.utils import sqlite3 from datasette.utils.asgi import Response from .base import BaseView, _error +from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, _analysis_is_write, @@ -221,10 +222,16 @@ class ExecuteWriteView(BaseView): execution_message=None, execution_links=None, execution_ok=None, + execute_write_returns_rows=False, + execute_write_columns=None, + execute_write_display_rows=None, + execute_write_truncated=False, status=200, ): parameter_values = parameter_values or {} execution_links = execution_links or [] + execute_write_columns = execute_write_columns or [] + execute_write_display_rows = execute_write_display_rows or [] parameter_names = [] analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) @@ -284,6 +291,10 @@ class ExecuteWriteView(BaseView): "execution_message": execution_message, "execution_links": execution_links, "execution_ok": execution_ok, + "execute_write_returns_rows": execute_write_returns_rows, + "execute_write_columns": execute_write_columns, + "execute_write_display_rows": execute_write_display_rows, + "execute_write_truncated": execute_write_truncated, "execute_disabled": bool(execute_disabled_reason), "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, @@ -355,11 +366,13 @@ class ExecuteWriteView(BaseView): status=ex.status, ) + wants_json = _wants_json(request, is_json, data) try: - cursor = await db.execute_write(sql, params, request=request) + execute_write_kwargs = {"request": request} + cursor = await db.execute_write(sql, params, **execute_write_kwargs) except sqlite3.DatabaseError as ex: message = str(ex) - if _wants_json(request, is_json, data): + if wants_json: return _block_framing(_error([message], 400)) return await self._render_form( request, @@ -378,17 +391,19 @@ class ExecuteWriteView(BaseView): message = "Query executed, {} row{} affected".format( cursor.rowcount, "" if cursor.rowcount == 1 else "s" ) - if _wants_json(request, is_json, data): - return _block_framing( - Response.json( - { - "ok": True, - "message": message, - "rowcount": cursor.rowcount, - "analysis": _analysis_rows(analysis), - } - ) - ) + if wants_json: + data = { + "ok": True, + "message": message, + "rowcount": cursor.rowcount, + "rows": [], + "truncated": False, + "analysis": _analysis_rows(analysis), + } + if cursor.description is not None: + data["rows"] = [dict(row) for row in cursor.fetchall()] + data["truncated"] = cursor.truncated + return _block_framing(Response.json(data)) inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor) execution_links = ( @@ -396,6 +411,20 @@ class ExecuteWriteView(BaseView): if inserted_row_url else [] ) + execute_write_returns_rows = cursor.description is not None + execute_write_columns = [] + execute_write_display_rows = [] + if execute_write_returns_rows: + execute_write_columns = [ + description[0] for description in cursor.description + ] + execute_write_display_rows = await display_query_rows( + self.ds, + db.name, + request, + cursor.fetchall(), + execute_write_columns, + ) return await self._render_form( request, db, @@ -405,6 +434,10 @@ class ExecuteWriteView(BaseView): execution_message=message, execution_links=execution_links, execution_ok=True, + execute_write_returns_rows=execute_write_returns_rows, + execute_write_columns=execute_write_columns, + execute_write_display_rows=execute_write_display_rows, + execute_write_truncated=cursor.truncated, ) diff --git a/docs/internals.rst b/docs/internals.rst index 4980ee8b..f269155a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1928,8 +1928,8 @@ Example usage: .. _database_execute_write: -await db.execute_write(sql, params=None, block=True) ----------------------------------------------------- +await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10) +-------------------------------------------------------------------------------------------------------- SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received. @@ -1937,7 +1937,30 @@ This method can be used to queue up a non-SELECT SQL query to be executed agains You can pass additional SQL parameters as a tuple or dictionary. -The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. +The optional ``request=`` argument is used internally by Datasette to pass request context to :ref:`write_wrapper plugin hooks `. + +The method will block until the operation is completed, and the return value will be an ``ExecuteWriteResult`` object. This imitates a subset of the ``sqlite3.Cursor`` object: + +``.rowcount`` + The number of rows modified by the statement, or ``-1`` if that number is unavailable. + +``.lastrowid`` + The row ID of the last modified row, as returned by ``sqlite3.Cursor.lastrowid``. + +``.description`` + The same column metadata exposed by Python's `sqlite3.Cursor.description `__: one tuple per returned column, or ``None`` if the statement does not return rows. + +``.truncated`` + ``True`` if the statement returned more rows than ``returning_limit``. + +``.fetchall()`` + Returns any rows buffered by Datasette from the statement, such as rows from SQLite's ``RETURNING`` clause. This may be limited by ``returning_limit`` unless ``return_all=True`` was used. This method empties the buffer, so calling it again will return an empty list. + +SQLite statements using ``RETURNING`` must have their rows consumed before the transaction can commit. Datasette will fetch up to ``returning_limit + 1`` rows before committing, store up to ``returning_limit`` rows on the result object and set ``.truncated`` if there were more. The default ``returning_limit`` is ``10``. + +When ``.truncated`` is ``True``, ``.rowcount`` will be ``-1``. SQLite only reports the final row count for a ``RETURNING`` statement after every returned row has been fetched, and Datasette has deliberately stopped fetching rows after ``returning_limit`` to avoid buffering a potentially large result in memory. + +If you need to retrieve every row returned by a statement, pass ``return_all=True``. This will buffer all returned rows in memory before committing. If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. diff --git a/docs/json_api.rst b/docs/json_api.rst index 4bd76717..65031bf4 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -554,7 +554,8 @@ Datasette analyzes the SQL before executing it. The actor must have ``execute-wr Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in arbitrary write SQL, and writes to SQLite virtual tables or shadow tables are rejected. SQL functions are allowed and are not separately restricted by Datasette permissions. -A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: +A successful response includes a message, the SQLite ``rowcount``, a ``"rows"`` +list, a ``"truncated"`` flag and a summary of the operations that were executed: The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. @@ -564,6 +565,8 @@ The shape of the ``"analysis"`` block is not yet considered a stable API and may "ok": true, "message": "Query executed, 1 row affected", "rowcount": 1, + "rows": [], + "truncated": false, "analysis": [ { "operation": "insert", @@ -577,6 +580,43 @@ The shape of the ``"analysis"`` block is not yet considered a stable API and may If SQLite reports ``-1`` for the row count, the message will be ``"Query executed"``. +For most write statements ``"rows"`` will be an empty list and ``"truncated"`` +will be ``false``. If the SQL uses SQLite's ``RETURNING`` clause, ``"rows"`` +will contain returned rows using the same default representation as table and +query JSON responses. ``"truncated"`` indicates if more rows were returned than +the execute-write returning row limit, which defaults to 10: + +.. code-block:: json + + { + "ok": true, + "message": "Query executed, 1 row affected", + "rowcount": 1, + "rows": [ + { + "id": 1, + "name": "Cleo" + } + ], + "truncated": false, + "analysis": [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row, update-row, delete-row", + "source": null + }, + { + "operation": "read", + "database": "data", + "table": "dogs", + "required_permission": "view-table", + "source": null + } + ] + } + Errors use the standard Datasette error format: .. code-block:: json diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 88f9d571..bb209649 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -5,15 +5,19 @@ Tests for the datasette.database.Database class import asyncio from types import SimpleNamespace from datasette.app import Datasette -from datasette.database import Database, Results, MultipleValues +from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues from datasette.database import DatasetteClosedError from datasette.database import _deliver_write_result -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, supports_returning from datasette.utils import Column import pytest import time import uuid +requires_sqlite_returning = pytest.mark.skipif( + not supports_returning(), reason="SQLite does not support RETURNING" +) + @pytest.fixture def db(app_client): @@ -469,13 +473,142 @@ async def test_view_names(db): @pytest.mark.asyncio async def test_execute_write_block_true(db): - await db.execute_write( + result = await db.execute_write( "update roadside_attractions set name = ? where pk = ?", ["Mystery!", 1] ) rows = await db.execute("select name from roadside_attractions where pk = 1") + assert result.rowcount == 1 + assert result.description is None + assert result.truncated is False + assert result.fetchall() == [] assert "Mystery!" == rows.rows[0][0] +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning(db): + await db.execute_write( + "create table write_returning (id integer primary key, name text)" + ) + result = await db.execute_write( + "insert into write_returning (name) values (?) returning id, name", + ["Cleo"], + ) + + assert result.rowcount == 1 + assert result.lastrowid == 1 + assert [column[0] for column in result.description] == ["id", "name"] + assert result.truncated is False + assert [dict(row) for row in result.fetchall()] == [{"id": 1, "name": "Cleo"}] + assert result.fetchall() == [] + assert (await db.execute("select id, name from write_returning")).dicts() == [ + {"id": 1, "name": "Cleo"} + ] + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_default_limit(db): + await db.execute_write( + "create table write_returning_limit (id integer primary key)" + ) + await db.execute_write_many( + "insert into write_returning_limit (id) values (?)", + [(i,) for i in range(1, 21)], + ) + + result = await db.execute_write( + "update write_returning_limit set id = id returning id" + ) + + assert result.rowcount == -1 + assert result.truncated is True + assert len(result.fetchall()) == 10 + assert ( + await db.execute("select count(*) from write_returning_limit") + ).single_value() == 20 + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_custom_limit(db): + await db.execute_write( + "create table write_returning_custom (id integer primary key)" + ) + await db.execute_write_many( + "insert into write_returning_custom (id) values (?)", + [(i,) for i in range(1, 6)], + ) + + result = await db.execute_write( + "update write_returning_custom set id = id returning id", + returning_limit=2, + ) + + assert result.rowcount == -1 + assert result.truncated is True + assert [row["id"] for row in result.fetchall()] == [1, 2] + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_exact_default_limit(db): + await db.execute_write( + "create table write_returning_exact_limit (id integer primary key)" + ) + await db.execute_write_many( + "insert into write_returning_exact_limit (id) values (?)", + [(i,) for i in range(1, 11)], + ) + + result = await db.execute_write( + "update write_returning_exact_limit set id = id returning id" + ) + + assert result.rowcount == 10 + assert result.truncated is False + assert len(result.fetchall()) == 10 + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_one_more_than_default_limit(db): + await db.execute_write( + "create table write_returning_one_more (id integer primary key)" + ) + await db.execute_write_many( + "insert into write_returning_one_more (id) values (?)", + [(i,) for i in range(1, 12)], + ) + + result = await db.execute_write( + "update write_returning_one_more set id = id returning id" + ) + + assert result.rowcount == -1 + assert result.truncated is True + assert len(result.fetchall()) == 10 + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_return_all(db): + await db.execute_write("create table write_returning_all (id integer primary key)") + await db.execute_write_many( + "insert into write_returning_all (id) values (?)", + [(i,) for i in range(1, 21)], + ) + + result = await db.execute_write( + "update write_returning_all set id = id returning id", + return_all=True, + ) + + assert result.rowcount == 20 + assert result.truncated is False + assert [row["id"] for row in result.fetchall()] == list(range(1, 21)) + + @pytest.mark.asyncio async def test_execute_write_block_false(db): await db.execute_write( @@ -487,6 +620,48 @@ async def test_execute_write_block_false(db): assert "Mystery!" == rows.rows[0][0] +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_with_returning_block_false(db): + await db.execute_write( + "create table write_returning_block_false (id integer primary key, name text)" + ) + task_id = await db.execute_write( + "insert into write_returning_block_false (name) values (?) returning id", + ["Cleo"], + block=False, + ) + + assert isinstance(task_id, uuid.UUID) + time.sleep(0.1) + assert ( + await db.execute("select name from write_returning_block_false") + ).single_value() == "Cleo" + + +def test_execute_write_result_closes_cursor_on_fetch_error(): + class Cursor: + description = (("id", None, None, None, None, None, None),) + lastrowid = 1 + rowcount = 0 + + def __init__(self): + self.closed = False + + def fetchmany(self, size): + raise sqlite3.DatabaseError("fetch failed") + + def close(self): + self.closed = True + + cursor = Cursor() + + with pytest.raises(sqlite3.DatabaseError): + ExecuteWriteResult.from_cursor(cursor) + + assert cursor.closed is True + + @pytest.mark.asyncio async def test_execute_write_script(db): await db.execute_write_script( diff --git a/tests/test_queries.py b/tests/test_queries.py index 89167a1d..cef06d7f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -8,6 +8,11 @@ from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden +from datasette.utils.sqlite import supports_returning + +requires_sqlite_returning = pytest.mark.skipif( + not supports_returning(), reason="SQLite does not support RETURNING" +) def _template_option_attributes(html, table): @@ -1884,10 +1889,144 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert allowed.status_code == 200 assert allowed.json()["ok"] is True assert allowed.json()["rowcount"] == 1 + assert allowed.json()["rows"] == [] + assert allowed.json()["truncated"] is False assert allowed.json()["analysis"][0]["operation"] == "insert" assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_json_includes_returning_rows(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_returning_json", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={ + "sql": "insert into dogs (name) values (:name) returning id, name", + "params": {"name": "Cleo"}, + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["message"] == "Query executed, 1 row affected" + assert data["rowcount"] == 1 + assert data["rows"] == [{"id": 1, "name": "Cleo"}] + assert data["truncated"] is False + assert [row["operation"] for row in data["analysis"]] == ["insert", "read"] + assert (await db.execute("select id, name from dogs")).dicts() == [ + {"id": 1, "name": "Cleo"} + ] + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_json_returning_rows_can_be_truncated(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_returning_json_truncated", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + for index in range(1, 12): + await db.execute_write( + "insert into dogs (name) values (?)", ["Dog {}".format(index)] + ) + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "update dogs set name = name || '!' returning id, name"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["message"] == "Query executed" + assert data["rowcount"] == -1 + assert data["rows"] == [ + {"id": index, "name": "Dog {}!".format(index)} for index in range(1, 11) + ] + assert data["truncated"] is True + assert (await db.execute("select count(*) from dogs where name like '%!'")).first()[ + 0 + ] == 11 + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_html_displays_returning_rows(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_returning_html", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "insert into dogs (name) values (:name) returning id, name", + "name": "Cleo", + }, + ) + non_returning_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={"sql": "insert into dogs (name) values ('Pancakes')"}, + ) + + assert response.status_code == 200 + assert "Query executed, 1 row affected" in response.text + assert "

    Returned rows

    " in response.text + assert '' in response.text + assert '' in response.text + assert '' in response.text + assert '' in response.text + assert '' in response.text + + assert non_returning_response.status_code == 200 + assert "Query executed, 1 row affected" in non_returning_response.text + assert "

    Returned rows

    " not in non_returning_response.text + assert '

    0 results

    ' not in non_returning_response.text + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_execute_write_html_returning_rows_can_be_truncated(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_returning_html_truncated", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + for index in range(1, 12): + await db.execute_write( + "insert into dogs (name) values (?)", ["Dog {}".format(index)] + ) + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={"sql": "update dogs set name = name || '!' returning id, name"}, + ) + + assert response.status_code == 200 + assert "

    Returned rows

    " in response.text + assert "Only the first 10 returned rows are shown." in response.text + assert '' in response.text + assert '' in response.text + assert '' in response.text + assert '' in response.text + assert '' not in response.text + assert '' not in response.text + + @pytest.mark.parametrize( "database_name, sql", ( @@ -3002,3 +3141,65 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): assert denied_response.status_code == 403 rows = (await db.execute("select name from dogs")).dicts() assert rows == [{"name": "Cleo"}] + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_stored_write_query_with_returning(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_write_returning", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (:name) returning id, name", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/insert_dog?_json=1", + actor={"id": "root"}, + data={"name": "Cleo"}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert (await db.execute("select id, name from dogs")).dicts() == [ + {"id": 1, "name": "Cleo"} + ] + + +@pytest.mark.asyncio +@requires_sqlite_returning +async def test_stored_write_query_with_truncated_returning_message(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_write_truncated_returning", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write_many( + "insert into dogs (name) values (?)", + [("Cleo",) for _ in range(20)], + ) + await ds.invoke_startup() + await ds.add_query( + "data", + "update_dogs", + "update dogs set name = name returning id", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/update_dogs?_json=1", + actor={"id": "root"}, + data={}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert response.json()["message"] == "Query executed" diff --git a/tests/test_utils.py b/tests/test_utils.py index f6de3b46..64607244 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,12 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request -from datasette.utils.sqlite import sqlite3, sqlite_hidden_table_names, sqlite_table_type +from datasette.utils.sqlite import ( + sqlite3, + sqlite_hidden_table_names, + sqlite_table_type, + supports_returning, +) import json import os import pathlib @@ -226,6 +231,20 @@ def test_detect_fts_different_table_names(table): conn.close() +def test_supports_returning(): + conn = utils.sqlite3.connect(":memory:") + try: + conn.execute("create table t (id integer primary key)") + conn.execute("insert into t default values returning id").fetchone() + expected = True + except sqlite3.DatabaseError: + expected = False + finally: + conn.close() + + assert supports_returning() is expected + + @pytest.mark.parametrize("use_fallback", (False, True)) def test_sqlite_table_type_detects_virtual_and_shadow_tables(monkeypatch, use_fallback): if use_fallback: From f9f346558265892d7cbc7c009eb590dece02c67b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2026 13:49:22 -0700 Subject: [PATCH 1826/1866] Better empty state message Root user was being told they didn't have permission when actually the problem was there were no tables at all. --- datasette/templates/execute_write.html | 2 +- tests/test_queries.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index a93de3a6..949850ed 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -111,7 +111,7 @@ form.sql.core input[data-execute-write-submit]:disabled { {% else %} -

    You don't currently have permission to insert, edit or delete from any tables.

    +

    There are no tables that you can currently edit.

    {% endif %}

    diff --git a/tests/test_queries.py b/tests/test_queries.py index cef06d7f..25e423d4 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1580,10 +1580,7 @@ async def test_execute_write_templates_are_filtered_by_permission_and_server_gen assert viewer_response.status_code == 200 assert "Start with a template" not in viewer_response.text - assert ( - "You don't currently have permission to insert, edit or delete from any tables." - in viewer_response.text - ) + assert "There are no tables that you can currently edit." in viewer_response.text assert "data-template-insert-sql" not in viewer_response.text assert "data-template-update-sql" not in viewer_response.text assert "data-template-delete-sql" not in viewer_response.text From 911954347e4ad55ba4f5cf6b576095299e3b76a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 31 May 2026 16:21:24 -0700 Subject: [PATCH 1827/1866] Release 1.0a32 Refs #2757, #2759, #2762, #2763 --- datasette/version.py | 2 +- docs/changelog.rst | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 76cabb1d..1e8c61d5 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a31" +__version__ = "1.0a32" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f9ffdbb..d5f8fa14 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,19 @@ Changelog ========= +.. _v1_0_a32: + +1.0a32 (2026-05-31) +------------------- + +SQLite INSERT ... RETURNING clauses are now supported by ``/db/-/execute-write``, plus several fixes relating to the :ref:`base_url setting `. + +- ``INSERT``/``UPDATE``/``DELETE`` statements that use SQLite's ``RETURNING`` clause now work correctly in the new ``/db/-/execute-write`` interface. Datasette fetches returned rows before committing the write transaction, displays them in the HTML UI and includes them in the ``"rows"`` key for the JSON API response. (:issue:`2762`, :pr:`2763`) +- ``Database.execute_write()`` now returns an ``ExecuteWriteResult`` object instead of the raw ``sqlite3.Cursor`` returned by ``conn.execute()``. The new object exposes ``.rowcount``, ``.lastrowid``, ``.description``, ``.truncated`` and ``.fetchall()``, and adds ``return_all=`` and ``returning_limit=`` options for controlling how rows from ``RETURNING`` statements are buffered. (:pr:`2763`) +- Fixed the ``/-/jump`` navigation search endpoint when Datasette is served with a configured ``base_url``. (:issue:`2757`) +- Fixed JSON and CSV export links, plus ``Link:`` alternate headers, on table, row and query pages when ``base_url`` is configured. These could previously be prefixed twice. (:issue:`2759`) +- Fixed several other ``base_url`` handling bugs, including the API explorer form actions and share links, the ``/-/patterns`` development page, permanent redirects such as ``/-`` to ``/-/`` and database query redirects from ``/?sql=...`` to ``//-/query?sql=...``. + .. _v1_0_a31: 1.0a31 (2026-05-28) From 6eaa9e31993d1b35a7252246aab31837a365ae56 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:19:47 -0700 Subject: [PATCH 1828/1866] Web UI to edit and delete stored queries (#2764) * Add web UI to edit and delete stored queries Stored query pages now offer Edit and Delete actions in the query actions menu, gated by the update-query and delete-query permissions. - New QueryEditView (GET/POST at ///-/edit) renders a pre-filled form for editing a query's title, description, SQL and privacy, reusing the create-query analysis UI. Changing the SQL still requires execute-sql; metadata-only edits do not. - QueryDeleteView gains a GET confirmation page and HTML form POST that redirects to the query list, while keeping the existing JSON API. - New default query_actions hook adds the Edit/Delete links for stored (non-config, non-trusted) queries the actor is allowed to manage. Permission semantics (already enforced by default_query_permissions_sql) are surfaced in the UI: owners can always edit/delete their queries; non-private queries can be edited/deleted by any actor with the relevant permission; private queries remain owner-only. Shared the create-query form styles into _query_form_styles.html so the edit form can reuse them. Animated demo: https://github.com/simonw/datasette/pull/2764#issuecomment-4655694668 Closes #2760 https://claude.ai/code/session_019GU9g3pZAERukLKYNa4uAL --- datasette/app.py | 5 + datasette/default_query_actions.py | 48 ++++ datasette/plugins.py | 1 + datasette/templates/_query_form_styles.html | 138 ++++++++++++ datasette/templates/query_create.html | 134 +---------- datasette/templates/query_delete.html | 82 +++++++ datasette/templates/query_edit.html | 133 +++++++++++ datasette/templates/query_list.html | 50 ++--- datasette/views/query_helpers.py | 29 +++ datasette/views/stored_queries.py | 169 +++++++++++++- docs/changelog.rst | 7 + docs/plugins.rst | 9 + docs/sql_queries.rst | 9 + tests/test_docs.py | 1 + tests/test_queries.py | 233 ++++++++++++++++++++ 15 files changed, 886 insertions(+), 162 deletions(-) create mode 100644 datasette/default_query_actions.py create mode 100644 datasette/templates/_query_form_styles.html create mode 100644 datasette/templates/query_delete.html create mode 100644 datasette/templates/query_edit.html diff --git a/datasette/app.py b/datasette/app.py index 8b8b601f..81d23acb 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -55,6 +55,7 @@ from .views.stored_queries import ( QueryCreateAnalyzeView, QueryDeleteView, QueryDefinitionView, + QueryEditView, GlobalQueryListView, QueryListView, QueryParametersView, @@ -2493,6 +2494,10 @@ class Datasette: QueryDefinitionView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", ) + add_route( + QueryEditView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/edit$", + ) add_route( QueryUpdateView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", diff --git a/datasette/default_query_actions.py b/datasette/default_query_actions.py new file mode 100644 index 00000000..2183e70b --- /dev/null +++ b/datasette/default_query_actions.py @@ -0,0 +1,48 @@ +from datasette import hookimpl +from datasette.resources import QueryResource + + +@hookimpl +def query_actions(datasette, actor, database, query_name, request): + # Only stored queries (with a name) can be edited or deleted + if not query_name: + return None + + async def inner(): + query = await datasette.get_query(database, query_name) + if query is None: + return [] + # Config-defined and trusted queries are managed outside the UI + if query.source == "config" or query.is_trusted: + return [] + + links = [] + if await datasette.allowed( + action="update-query", + resource=QueryResource(database, query_name), + actor=actor, + ): + links.append( + { + "href": datasette.urls.table(database, query_name) + "/-/edit", + "label": "Edit this query", + "description": ( + "Change the title, description, SQL or visibility." + ), + } + ) + if await datasette.allowed( + action="delete-query", + resource=QueryResource(database, query_name), + actor=actor, + ): + links.append( + { + "href": datasette.urls.table(database, query_name) + "/-/delete", + "label": "Delete this query", + "description": "Permanently remove this saved query.", + } + ) + return links + + return inner diff --git a/datasette/plugins.py b/datasette/plugins.py index 5a31cdad..f0fbc7f8 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -31,6 +31,7 @@ DEFAULT_PLUGINS = ( "datasette.default_debug_menu", "datasette.default_jump_items", "datasette.default_database_actions", + "datasette.default_query_actions", "datasette.handle_exception", "datasette.forbidden", "datasette.events", diff --git a/datasette/templates/_query_form_styles.html b/datasette/templates/_query_form_styles.html new file mode 100644 index 00000000..cf2dd42c --- /dev/null +++ b/datasette/templates/_query_form_styles.html @@ -0,0 +1,138 @@ + diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index ec910456..f2016f27 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -6,139 +6,7 @@ {{- super() -}} {% include "_codemirror.html" %} {% include "_execute_write_analysis_styles.html" %} - +{% include "_query_form_styles.html" %} {% endblock %} {% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} diff --git a/datasette/templates/query_delete.html b/datasette/templates/query_delete.html new file mode 100644 index 00000000..4d0699a7 --- /dev/null +++ b/datasette/templates/query_delete.html @@ -0,0 +1,82 @@ +{% extends "base.html" %} + +{% block title %}Delete query: {{ query.name }}{% endblock %} + +{% block extra_head %} +{{- super() -}} + +{% endblock %} + +{% block body_class %}query-delete db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +
    + +

    Delete query: {{ query.title or query.name }}

    + +

    Are you sure you want to delete this saved query? This cannot be undone.

    + +
    +
    URL
    +
    {{ query_url }}
    + {% if query.description %} +
    Description
    +
    {{ query.description }}
    + {% endif %} +
    SQL
    + 
    {{ query.sql }}
    +
    + + +

    + + Cancel +

    + + +
    + +{% endblock %} diff --git a/datasette/templates/query_edit.html b/datasette/templates/query_edit.html new file mode 100644 index 00000000..3eadf42a --- /dev/null +++ b/datasette/templates/query_edit.html @@ -0,0 +1,133 @@ +{% extends "base.html" %} + +{% block title %}Edit query: {{ name }}{% endblock %} + +{% block extra_head %} +{{- super() -}} +{% include "_codemirror.html" %} +{% include "_execute_write_analysis_styles.html" %} +{% include "_query_form_styles.html" %} +{% endblock %} + +{% block body_class %}query-edit db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +
    + +

    Edit query: {{ title or name }}

    + +
    +
    +

    +

    {{ query_url }}

    +

    +
    + +

    + +

    + {% if analysis_error %}This query cannot be saved until the SQL is valid.{% elif not has_sql %}Enter SQL to analyze this query.{% elif analysis_is_write %}This query updates data in the database.{% else %}This is a read-only query.{% endif %} + + + Queries marked private can only be seen and edited by you, their owner. +

    +

    Cancel

    + +
    + {% if has_sql %} +

    Query operations

    + {% if analysis_error %} +

    {{ analysis_error }}

    + {% elif analysis_rows %} +
    idname1Cleo1Dog 1!10Dog 10!11Dog 11!
    + + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + {% endfor %} + +
    OperationDatabaseTableRequired permissionAllowed
    {{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% else %}n/a{% endif %}{% if row.allowed is none %}n/a{% elif row.allowed %}yes{% else %}no{% endif %}
    + {% else %} +

    Analysis will show each affected table and required permission.

    + {% endif %} + {% endif %} + + + + + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% endblock %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index fa4859b1..a8c9a391 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -205,32 +205,32 @@

    Queries

    -
    -

    - - - {% if filters.is_write %}{% endif %} - {% if filters.is_private %}{% endif %} - {% if filters.source %}{% endif %} - {% if filters.owner_id %}{% endif %} - -

    -
    - - - {% if queries %} +
    +

    + + + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %} + +

    +
    + + +
    diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index f30a30bc..9efe3f81 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -436,6 +436,35 @@ async def _query_create_form_context( } +async def _query_edit_form_context( + datasette, + request, + db, + existing: StoredQuery, + *, + sql=None, + title=None, + description=None, + is_private=None, +): + sql = existing.sql if sql is None else sql + title = existing.title if title is None else title + description = existing.description if description is None else description + is_private = existing.is_private if is_private is None else is_private + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "name": existing.name, + "sql": sql, + "title": title or "", + "description": description or "", + "is_private": is_private, + "query_url": datasette.urls.table(db.name, existing.name), + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 8c4e849e..2753f876 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -18,6 +18,7 @@ from .query_helpers import ( _query_create_analysis_data, _query_create_form_context, _query_create_form_error_message, + _query_edit_form_context, _query_list_limit, ) @@ -464,13 +465,164 @@ class QueryUpdateView(BaseView): return Response.json({"ok": True}) -class QueryDeleteView(BaseView): - name = "query-delete" +class QueryEditView(BaseView): + name = "query-edit" + has_json_alternate = False - async def post(self, request): + async def _load(self, request): db = await self.ds.resolve_database(request) query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def _render_form( + self, + request, + db, + existing, + *, + sql=None, + title=None, + description=None, + is_private=None, + status=200, + ): + response = await self.render( + ["query_edit.html"], + request, + await _query_edit_form_context( + self.ds, + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + return await self._render_form(request, db, existing) + + async def post(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + return _error(["Invalid form submission"], 400) + sql = data.get("sql") + sql = existing.sql if sql is None else sql.strip() + title = data.get("title") or "" + description = data.get("description") or "" + is_private = _as_bool(data.get("is_private")) + + update = { + "title": title, + "description": description, + "is_private": is_private, + } + if sql != existing.sql: + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + self.ds.add_message( + request, + "Permission denied: need execute-sql to change the SQL", + self.ds.ERROR, + ) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=403, + ) + update["sql"] = sql + + try: + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + self.ds.add_message(request, ex.message, self.ds.ERROR) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=ex.status, + ) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + self.ds.add_message(request, "Query updated", self.ds.INFO) + return Response.redirect( + self.ds.urls.path(self.ds.urls.table(db.name, query_name)) + ) + + +class QueryDeleteView(BaseView): + name = "query-delete" + has_json_alternate = False + + async def _load(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + return await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, + ) + + async def post(self, request): + db, query_name, existing = await self._load(request) if existing is None: return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( @@ -479,5 +631,14 @@ class QueryDeleteView(BaseView): actor=request.actor, ): return _error(["Permission denied: need delete-query"], 403) + + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) - return Response.json({"ok": True}) + if is_json: + return Response.json({"ok": True}) + self.ds.add_message( + request, + "Query “{}” deleted".format(existing.title or query_name), + self.ds.INFO, + ) + return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/docs/changelog.rst b/docs/changelog.rst index d5f8fa14..75e4f3e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a33: + +1.0a33 (unreleased) +------------------- + +- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + .. _v1_0_a32: 1.0a32 (2026-05-31) diff --git a/docs/plugins.rst b/docs/plugins.rst index d578e9e2..c2eb282a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -271,6 +271,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_token_handler" ] }, + { + "name": "datasette.default_query_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "query_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index c0ba67f0..371348fb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -142,6 +142,15 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. +Editing and deleting stored queries ++++++++++++++++++++++++++++++++++++ + +The page for a stored query includes a "Query actions" menu with **Edit this query** and **Delete this query** links for actors who have permission to use them. + +The owner of a stored query can always edit and delete it. For queries that are not private, any actor granted the ``update-query`` or ``delete-query`` permission can edit or delete the query, even if they did not create it. Private queries can only be edited or deleted by their owner, regardless of any broad permission grants. + +Editing a query lets you change its title, description, SQL and whether it is private. Changing the SQL also requires the ``execute-sql`` permission (and the relevant write permissions for writable queries). The same operations are available through the JSON API by sending a ``POST`` to ``///-/update`` or ``///-/delete``. Trusted stored queries cannot be edited or deleted through the web interface or the JSON API. + Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: diff --git a/tests/test_docs.py b/tests/test_docs.py index 9cf39f41..51caf595 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -77,6 +77,7 @@ def documented_views(): "QueryCreateAnalyzeView", "QueryDeleteView", "QueryDefinitionView", + "QueryEditView", "QueryListView", "QueryParametersView", "QueryStoreView", diff --git a/tests/test_queries.py b/tests/test_queries.py index 25e423d4..6e9bcbdb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -3,6 +3,7 @@ import re from html import unescape import pytest +from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource @@ -712,6 +713,10 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries?is_private=1", actor={"id": "root"}, ) + no_results_response = await ds.client.get( + "/data/-/queries?q=nope", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text @@ -799,6 +804,13 @@ async def test_query_list_search_filter_and_html(): 'Not private0' not in filtered_private_response.text ) + assert no_results_response.status_code == 200 + assert "No queries found." in no_results_response.text + assert 'class="query-list-filters core"' not in no_results_response.text + assert 'id="query-search"' not in no_results_response.text + assert 'class="query-list-facets"' not in no_results_response.text + assert "

    Mode

    " not in no_results_response.text + assert "

    Visibility

    " not in no_results_response.text @pytest.mark.asyncio @@ -1114,6 +1126,227 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo assert query.title == "Internal" +async def _make_ds_with_user_query(name, *, is_private=False, owner_id="owner"): + ds = Datasette(memory=True, settings={"default_allow_sql": True}) + db = ds.add_memory_database(name, name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Saved query", + description="A saved query", + source="user", + owner_id=owner_id, + is_private=is_private, + ) + return ds + + +@pytest.mark.asyncio +async def test_query_edit_form_renders_and_updates_for_owner(): + ds = await _make_ds_with_user_query("query_edit_owner") + actor = {"id": "owner"} + + # GET renders the form pre-filled with existing values + get_response = await ds.client.get("/data/saved/-/edit", actor=actor) + assert get_response.status_code == 200 + assert 'value="Saved query"' in get_response.text + assert ">A saved query" in get_response.text + assert "select * from dogs" in get_response.text + # URL slug is shown but not editable + assert 'name="name"' not in get_response.text + + # POST updates the query and redirects back to the query page + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Updated title", + "description": "Updated description", + "sql": "select id from dogs", + "is_private": "1", + }, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data/saved" + + query = await ds.get_query("data", "saved") + assert query.title == "Updated title" + assert query.description == "Updated description" + assert query.sql == "select id from dogs" + assert query.is_private is True + + +@pytest.mark.asyncio +async def test_query_edit_metadata_only_does_not_require_execute_sql(): + # An owner who can no longer execute SQL can still edit title/description + ds = await _make_ds_with_user_query("query_edit_metadata_only") + actor = {"id": "owner"} + + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Renamed", + "description": "A saved query", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Renamed" + + +@pytest.mark.asyncio +async def test_private_query_edit_delete_restricted_to_owner(): + ds = await _make_ds_with_user_query( + "query_edit_private", is_private=True, owner_id="owner" + ) + + # A different actor cannot view, edit or delete the private query + other = {"id": "intruder"} + assert (await ds.client.get("/data/saved/-/edit", actor=other)).status_code == 403 + assert (await ds.client.get("/data/saved/-/delete", actor=other)).status_code == 403 + delete_attempt = await ds.client.post( + "/data/saved/-/delete", + actor=other, + data={}, + ) + assert delete_attempt.status_code == 403 + assert await ds.get_query("data", "saved") is not None + + # The owner can edit and delete + owner = {"id": "owner"} + assert (await ds.client.get("/data/saved/-/edit", actor=owner)).status_code == 200 + + +@pytest.mark.asyncio +async def test_non_private_query_editable_by_permitted_non_owner(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + "delete-query": {"id": "editor"}, + } + } + } + }, + ) + db = ds.add_memory_database("query_non_private_editor", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Shared", + source="user", + owner_id="owner", + is_private=False, + ) + + editor = {"id": "editor"} + # Editor (not the owner) can edit because the query is not private + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=editor, + data={ + "title": "Edited by editor", + "description": "", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Edited by editor" + + # Editor can also delete it + delete_response = await ds.client.post( + "/data/saved/-/delete", + actor=editor, + data={}, + ) + assert delete_response.status_code == 302 + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_delete_confirmation_and_form_delete(): + ds = await _make_ds_with_user_query("query_delete_form") + actor = {"id": "owner"} + + get_response = await ds.client.get("/data/saved/-/delete", actor=actor) + assert get_response.status_code == 200 + assert "Are you sure" in get_response.text + assert "select * from dogs" in get_response.text + soup = Soup(get_response.text, "html.parser") + form = soup.select_one("form.query-delete-form") + assert form is not None + assert "core" in form["class"] + assert form.select_one('input[type="submit"][value="Delete query"]') is not None + + post_response = await ds.client.post( + "/data/saved/-/delete", + actor=actor, + data={}, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data" + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_action_menu_shows_edit_and_delete_for_owner(): + ds = await _make_ds_with_user_query("query_action_menu") + + owner_response = await ds.client.get("/data/saved", actor={"id": "owner"}) + assert owner_response.status_code == 200 + assert "/data/saved/-/edit" in owner_response.text + assert "/data/saved/-/delete" in owner_response.text + + # A different actor (the query is public) cannot edit/delete by default + other_response = await ds.client.get("/data/saved", actor={"id": "stranger"}) + assert other_response.status_code == 200 + assert "/data/saved/-/edit" not in other_response.text + assert "/data/saved/-/delete" not in other_response.text + + +@pytest.mark.asyncio +async def test_query_edit_rejected_for_trusted_query(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + }, + "queries": {"trusted_report": {"sql": "select 1 as one"}}, + } + } + }, + ) + ds.add_memory_database("query_edit_trusted", name="data") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/trusted_report/-/edit", actor={"id": "editor"} + ) + assert response.status_code == 403 + # Edit/delete links should not appear on a trusted/config query page + page = await ds.client.get("/data/trusted_report", actor={"id": "editor"}) + assert "/data/trusted_report/-/edit" not in page.text + + @pytest.mark.asyncio async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) From 03f1ffdf8fbf0ed7da46be48f3fcc3f4698e1e21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:45:01 -0700 Subject: [PATCH 1829/1866] Centralize JSON extra parsing --- datasette/extras.py | 6 ++++++ datasette/renderer.py | 3 ++- datasette/views/table.py | 7 ++----- tests/test_table_api.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 datasette/extras.py diff --git a/datasette/extras.py b/datasette/extras.py new file mode 100644 index 00000000..01a9fb4b --- /dev/null +++ b/datasette/extras.py @@ -0,0 +1,6 @@ +def extra_names_from_request(request): + extra_bits = request.args.getlist("_extra") + extras = set() + for bit in extra_bits: + extras.update(part for part in bit.split(",") if part) + return extras diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,4 +1,5 @@ import json +from datasette.extras import extra_names_from_request from datasette.utils import ( value_as_boolean, remove_infinites, @@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None): # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 - if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + if isinstance(data, dict) and "columns" not in extra_names_from_request(request): data.pop("columns", None) # Handle _nl option for _shape=array diff --git a/datasette/views/table.py b/datasette/views/table.py index 4df1e1b4..9ba249f4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -6,6 +6,7 @@ import urllib from asyncinject import Registry import markupsafe +from datasette.extras import extra_names_from_request from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.events import ( @@ -850,11 +851,7 @@ class TableDropView(BaseView): def _get_extras(request): - extra_bits = request.args.getlist("_extra") - extras = set() - for bit in extra_bits: - extras.update(bit.split(",")) - return extras + return extra_names_from_request(request) async def _columns_to_select(table_columns, pks, request): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ceeb646d..eeb3dc8b 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1376,6 +1376,17 @@ async def test_table_extras(ds_client, extra, expected_json): assert response.json() == expected_json +@pytest.mark.asyncio +async def test_table_extra_columns_can_be_comma_separated(ds_client): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=columns,count" + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["id", "content", "content2"] + assert data["count"] == 1 + + @pytest.mark.asyncio async def test_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook""" From 17bbe6855c34630c14b077e08247d453d371cdea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:52:10 -0700 Subject: [PATCH 1830/1866] Refactor table JSON extras into classes --- datasette/extras.py | 94 ++++ datasette/views/table.py | 584 ++----------------------- datasette/views/table_extras.py | 746 ++++++++++++++++++++++++++++++++ 3 files changed, 884 insertions(+), 540 deletions(-) create mode 100644 datasette/views/table_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index 01a9fb4b..786ec4f4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,6 +1,100 @@ +import re +from enum import Enum +from typing import ClassVar + +from asyncinject import Registry + + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") extras = set() for bit in extra_bits: extras.update(part for part in bit.split(",") if part) return extras + + +class ExtraScope(Enum): + TABLE = "table" + + +class Provider: + name: ClassVar[str | None] = None + scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + public: ClassVar[bool] = False + + @classmethod + def key(cls): + return cls.name or _camel_to_snake(cls.__name__) + + @classmethod + def available_for(cls, scope): + return scope in cls.scopes + + async def resolve(self, context): + raise NotImplementedError + + +class Extra(Provider): + description: ClassVar[str | None] = None + public: ClassVar[bool] = True + stable: ClassVar[bool] = True + expensive: ClassVar[bool] = False + docs_note: ClassVar[str | None] = None + + @classmethod + def documentation(cls): + return { + "name": cls.key(), + "description": cls.description, + "scopes": [ + scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) + ], + "stable": cls.stable, + "expensive": cls.expensive, + "docs_note": cls.docs_note, + } + + +class ExtraRegistry: + def __init__(self, classes): + self.classes = list(classes) + self.classes_by_name = {cls.key(): cls for cls in self.classes} + + def classes_for_scope(self, scope, include_internal=True): + classes = [ + cls + for cls in self.classes + if cls.available_for(scope) and (include_internal or cls.public) + ] + return classes + + def public_classes_for_scope(self, scope): + return self.classes_for_scope(scope, include_internal=False) + + async def resolve(self, requested, context, scope): + registry = Registry() + + async def context_provider(): + return context + + registry.register(context_provider, name="context") + + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + + public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} + requested_public_names = [ + name + for name in requested + if name in public_names and name in registry._registry + ] + resolved = await registry.resolve_multi(requested_public_names) + return { + name: resolved[name] for name in requested_public_names if name in resolved + } + + +def _camel_to_snake(name): + name = re.sub(r"(Extra|Provider)$", "", name) + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() diff --git a/datasette/views/table.py b/datasette/views/table.py index 9ba249f4..c2d520f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,12 +3,10 @@ import itertools import json import urllib -from asyncinject import Registry import markupsafe from datasette.extras import extra_names_from_request from datasette.plugins import pm -from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, @@ -47,6 +45,12 @@ from datasette.filters import Filters import sqlite_utils from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView +from .table_extras import ( + TABLE_EXTRA_BUNDLES, + TableExtraContext, + resolve_table_extras, + table_extra_registry, +) LINK_WITH_LABEL = ( '{label} {id}' @@ -1465,560 +1469,60 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) - async def extra_count_sql(): - return count_sql - - async def extra_count(): - "Total count of rows matching these filters" - # Calculate the total count for this query - count = None - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - # We can use a previously cached table row count - try: - count = datasette.inspect_data[database_name]["tables"][table_name][ - "count" - ] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and count is None and not nocount: - count_sql_limited = ( - f"select count(*) from (select * {from_sql} limit 10001)" - ) - try: - count_rows = list(await db.execute(count_sql_limited, from_sql_params)) - count = count_rows[0][0] - except QueryInterrupted: - pass - return count - - async def facet_instances(extra_count): - facet_instances = [] - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - for facet_class in facet_classes: - facet_instances.append( - facet_class( - datasette, - request, - database_name, - sql=sql_no_order_no_limit, - params=params, - table=table_name, - table_config=table_metadata, - row_count=extra_count, - ) - ) - return facet_instances - - async def extra_facet_results(facet_instances): - "Results of facets calculated against this data" - facet_results = {} - facets_timed_out = [] - - if not nofacet: - # Run them in parallel - facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await run_sequential(*facet_awaitables) - for ( - instance_facet_results, - instance_facets_timed_out, - ) in facet_awaitable_results: - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - return { - "results": facet_results, - "timed_out": facets_timed_out, - } - - async def extra_suggested_facets(facet_instances): - "Suggestions for facets that might return interesting results" - suggested_facets = [] - # Calculate suggested facets - if ( - datasette.setting("suggest_facets") - and datasette.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - # Run them in parallel - facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await run_sequential(*facet_suggest_awaitables): - suggested_facets.extend(suggest_result) - return suggested_facets - # Faceting if not datasette.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") - # human_description_en combines filters AND search, if provided - async def extra_human_description_en(): - "Human-readable description of the filters" - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - if sort or sort_desc: - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - return human_description_en - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - - async def extra_next_url(): - "Full URL for the next page of results" - return next_url - - async def extra_columns(): - "Column names returned by this query" - return columns - - async def extra_all_columns(): - "All columns in the table, regardless of _col/_nocol filtering" - return list(table_columns) - - async def extra_primary_keys(): - "Primary keys for this table" - return pks - - async def extra_actions(): - async def actions(): - links = [] - kwargs = { - "datasette": datasette, - "database": database_name, - "actor": request.actor, - "request": request, - } - if is_view: - kwargs["view"] = table_name - method = pm.hook.view_actions - else: - kwargs["table"] = table_name - method = pm.hook.table_actions - for hook in method(**kwargs): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - return actions - - async def extra_is_view(): - return is_view - - async def extra_debug(): - "Extra debug information" - return { - "resolved": repr(resolved), - "url_vars": request.url_vars, - "nofacet": nofacet, - "nosuggest": nosuggest, - } - - async def extra_request(): - "Full information about the request" - return { - "url": request.url, - "path": request.path, - "full_path": request.full_path, - "host": request.host, - "args": request.args._data, - } - - async def run_display_columns_and_rows(): - display_columns, display_rows = await display_columns_and_rows( - datasette, - database_name, - table_name, - results.description, - rows, - link_column=not is_view, - truncate_cells=datasette.setting("truncate_cells_html"), - sortable_columns=sortable_columns, - request=request, - ) - return { - "columns": display_columns, - "rows": display_rows, - } - - async def extra_display_columns(run_display_columns_and_rows): - return run_display_columns_and_rows["columns"] - - async def extra_display_rows(run_display_columns_and_rows): - return run_display_columns_and_rows["rows"] - - async def extra_render_cell(): - "Rendered HTML for each cell using the render_cell plugin hook" - pks_for_display = pks if pks else (["rowid"] if not is_view else []) - col_names = [col[0] for col in results.description] - ct_map = await datasette.get_column_types(database_name, table_name) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, col_names): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table_name, - database=database_name, - datasette=datasette, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table_name, - pks=pks_for_display, - database=database_name, - datasette=datasette, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - return rendered_rows - - async def extra_query(): - "Details of the underlying SQL query" - return { - "sql": sql, - "params": params, - } - - async def extra_column_types(): - "Column type assignments for this table" - ct_map = await datasette.get_column_types(database_name, table_name) - return { - col_name: { - "type": ct.name, - "config": ct.config, - } - for col_name, ct in ct_map.items() - } - - async def extra_set_column_type_ui(): - "Column type UI metadata for this table" - if is_view: - return None - - if not await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await datasette._get_resource_column_details( - database_name, table_name - ) - ct_map = await datasette.get_column_types(database_name, table_name) - columns = {} - for column_name, column_detail in column_details.items(): - current = ct_map.get(column_name) - columns[column_name] = { - "current": ( - {"type": current.name, "config": current.config} - if current is not None - else None - ), - "options": [ - { - "name": name, - "description": ct_cls.description, - } - for name, ct_cls in sorted(datasette._column_types.items()) - if datasette._column_type_is_applicable(ct_cls, column_detail) - ], - } - return { - "path": "{}/-/set-column-type".format( - datasette.urls.table(database_name, table_name) - ), - "columns": columns, - } - - async def extra_metadata(): - "Metadata about the table and database" - tablemetadata = await datasette.get_resource_metadata(database_name, table_name) - - rows = await datasette.get_internal_database().execute( - """ - SELECT - column_name, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND key = 'description' - """, - [database_name, table_name], - ) - tablemetadata["columns"] = dict(rows) - return tablemetadata - - async def extra_database(): - return database_name - - async def extra_table(): - return table_name - - async def extra_database_color(): - return db.color - - async def extra_form_hidden_args(): - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_sort_desc", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - return form_hidden_args - - async def extra_filters(): - return filters - - async def extra_custom_table_templates(): - return [ - f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - "_table.html", - ] - - async def extra_sorted_facet_results(extra_facet_results): - facet_configs = table_metadata.get("facets", []) - if facet_configs: - # Build ordered list of facet names from metadata config - metadata_facet_names = [] - for fc in facet_configs: - if isinstance(fc, str): - metadata_facet_names.append(fc) - elif isinstance(fc, dict): - metadata_facet_names.append(list(fc.values())[0]) - metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} - metadata_facets = [] - request_facets = [] - for f in extra_facet_results["results"].values(): - if f["name"] in metadata_order: - metadata_facets.append(f) - else: - request_facets.append(f) - metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) - request_facets.sort( - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - return metadata_facets + request_facets - else: - return sorted( - extra_facet_results["results"].values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - - async def extra_table_definition(): - return await db.get_table_definition(table_name) - - async def extra_view_definition(): - return await db.get_view_definition(table_name) - - async def extra_renderers(extra_expandable_columns, extra_query): - renderers = {} - url_labels_extra = {} - if extra_expandable_columns: - url_labels_extra = {"_labels": "on"} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=columns or [], - rows=rows or [], - sql=extra_query.get("sql", None), - query_name=None, - database=database_name, - table=table_name, - request=request, - view_name="table", - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - return renderers - - async def extra_private(): - return private - - async def extra_expandable_columns(): - expandables = [] - db = datasette.databases[database_name] - for fk in await db.foreign_keys_for_table(table_name): - label_column = await db.label_column_for_table(fk["other_table"]) - expandables.append((fk, label_column)) - return expandables - - async def extra_extras(): - "Available ?_extra= blocks" - all_extras = [ - (key[len("extra_") :], fn.__doc__) - for key, fn in registry._registry.items() - if key.startswith("extra_") - ] - return [ - { - "name": name, - "description": doc, - "toggle_url": datasette.absolute_url( - request, - datasette.urls.path( - path_with_added_args(request, {"_extra": name}) - if name not in extras - else path_with_removed_args(request, {"_extra": name}) - ), - ), - "selected": name in extras, - } - for name, doc in all_extras - ] - - async def extra_facets_timed_out(extra_facet_results): - return extra_facet_results["timed_out"] - - bundles = { - "html": [ - "suggested_facets", - "facet_results", - "facets_timed_out", - "count", - "count_sql", - "human_description_en", - "next_url", - "metadata", - "query", - "columns", - "display_columns", - "display_rows", - "database", - "table", - "database_color", - "actions", - "filters", - "renderers", - "custom_table_templates", - "sorted_facet_results", - "table_definition", - "view_definition", - "is_view", - "private", - "primary_keys", - "all_columns", - "expandable_columns", - "form_hidden_args", - "set_column_type_ui", - ] - } - - for key, values in bundles.items(): + for key, values in TABLE_EXTRA_BUNDLES.items(): if f"_{key}" in extras: extras.update(values) extras.discard(f"_{key}") - registry = Registry( - extra_count, - extra_count_sql, - extra_facet_results, - extra_facets_timed_out, - extra_suggested_facets, - facet_instances, - extra_human_description_en, - extra_next_url, - extra_columns, - extra_all_columns, - extra_primary_keys, - run_display_columns_and_rows, - extra_display_columns, - extra_display_rows, - extra_render_cell, - extra_debug, - extra_request, - extra_query, - extra_column_types, - extra_set_column_type_ui, - extra_metadata, - extra_extras, - extra_database, - extra_table, - extra_database_color, - extra_actions, - extra_filters, - extra_renderers, - extra_custom_table_templates, - extra_sorted_facet_results, - extra_table_definition, - extra_view_definition, - extra_is_view, - extra_private, - extra_expandable_columns, - extra_form_hidden_args, + table_extra_context = TableExtraContext( + datasette=datasette, + request=request, + resolved=resolved, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + table_columns=table_columns, + pks=pks, + count_sql=count_sql, + from_sql=from_sql, + from_sql_params=from_sql_params, + nocount=nocount, + nofacet=nofacet, + nosuggest=nosuggest, + next_arg=request.args.get("_next"), + next_value=next_value, + next_url=next_url, + sql=sql, + sql_no_order_no_limit=sql_no_order_no_limit, + params=params, + table_metadata=table_metadata, + filters=filters, + extra_human_descriptions=extra_human_descriptions, + sort=sort, + sort_desc=sort_desc, + sortable_columns=sortable_columns, + extras=extras, + extra_registry=table_extra_registry, + display_columns_and_rows=display_columns_and_rows, + run_sequential=run_sequential, ) - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) data = { "ok": True, "next": next_value and str(next_value) or None, } - data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } - ) + data.update(await resolve_table_extras(extras, table_extra_context)) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py new file mode 100644 index 00000000..2ec2adf0 --- /dev/null +++ b/datasette/views/table_extras.py @@ -0,0 +1,746 @@ +import itertools +from dataclasses import dataclass + +from datasette.database import QueryInterrupted +from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.plugins import pm +from datasette.resources import TableResource +from datasette.utils import ( + await_me_maybe, + call_with_supported_arguments, + path_with_added_args, + path_with_format, + path_with_removed_args, + to_css_class, +) + + +@dataclass(frozen=True) +class TableExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + is_view: bool + private: bool + rows: list + columns: list + results_description: list + table_columns: list + pks: list + count_sql: str + from_sql: str + from_sql_params: dict + nocount: object + nofacet: object + nosuggest: object + next_arg: object + next_value: object + next_url: str | None + sql: str + sql_no_order_no_limit: str + params: dict + table_metadata: dict + filters: object + extra_human_descriptions: list + sort: str | None + sort_desc: str | None + sortable_columns: set + extras: set + extra_registry: ExtraRegistry + display_columns_and_rows: object + run_sequential: object + + +class CountSqlExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.count_sql + + +class CountExtra(Extra): + description = "Total count of rows matching these filters" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context): + count = None + if ( + not context.db.is_mutable + and context.datasette.inspect_data + and context.count_sql == f"select count(*) from {context.table_name} " + ): + try: + count = context.datasette.inspect_data[context.database_name]["tables"][ + context.table_name + ]["count"] + except KeyError: + pass + + if context.count_sql and count is None and not context.nocount: + count_sql_limited = ( + f"select count(*) from (select * {context.from_sql} limit 10001)" + ) + try: + count_rows = list( + await context.db.execute(count_sql_limited, context.from_sql_params) + ) + count = count_rows[0][0] + except QueryInterrupted: + pass + return count + + +class FacetInstancesProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, count): + facet_instances = [] + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + for facet_class in facet_classes: + facet_instances.append( + facet_class( + context.datasette, + context.request, + context.database_name, + sql=context.sql_no_order_no_limit, + params=context.params, + table=context.table_name, + table_config=context.table_metadata, + row_count=count, + ) + ) + return facet_instances + + +class FacetResultsExtra(Extra): + description = "Results of facets calculated against this data" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + facet_results = {} + facets_timed_out = [] + + if not context.nofacet: + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await context.run_sequential(*facet_awaitables) + for ( + instance_facet_results, + instance_facets_timed_out, + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + return { + "results": facet_results, + "timed_out": facets_timed_out, + } + + +class FacetsTimedOutExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + return facet_results["timed_out"] + + +class SuggestedFacetsExtra(Extra): + description = "Suggestions for facets that might return interesting results" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + suggested_facets = [] + if ( + context.datasette.setting("suggest_facets") + and context.datasette.setting("allow_facet") + and not context.next_arg + and not context.nofacet + and not context.nosuggest + ): + facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] + for suggest_result in await context.run_sequential( + *facet_suggest_awaitables + ): + suggested_facets.extend(suggest_result) + return suggested_facets + + +class HumanDescriptionEnExtra(Extra): + description = "Human-readable description of the filters" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + human_description_en = context.filters.human_description_en( + extra=context.extra_human_descriptions + ) + if context.sort or context.sort_desc: + sorted_by = "sorted by {}{}".format( + (context.sort or context.sort_desc), + " descending" if context.sort_desc else "", + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + return human_description_en + + +class NextUrlExtra(Extra): + description = "Full URL for the next page of results" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.next_url + + +class ColumnsExtra(Extra): + description = "Column names returned by this query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.columns + + +class AllColumnsExtra(Extra): + description = "All columns in the table, regardless of _col/_nocol filtering" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return list(context.table_columns) + + +class PrimaryKeysExtra(Extra): + description = "Primary keys for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.pks + + +class ActionsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + async def actions(): + links = [] + kwargs = { + "datasette": context.datasette, + "database": context.database_name, + "actor": context.request.actor, + "request": context.request, + } + if context.is_view: + kwargs["view"] = context.table_name + method = pm.hook.view_actions + else: + kwargs["table"] = context.table_name + method = pm.hook.table_actions + for hook in method(**kwargs): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + return actions + + +class IsViewExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.is_view + + +class DebugExtra(Extra): + description = "Extra debug information" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "resolved": repr(context.resolved), + "url_vars": context.request.url_vars, + "nofacet": context.nofacet, + "nosuggest": context.nosuggest, + } + + +class RequestExtra(Extra): + description = "Full information about the request" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "url": context.request.url, + "path": context.request.path, + "full_path": context.request.full_path, + "host": context.request.host, + "args": context.request.args._data, + } + + +class DisplayColumnsAndRowsProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + display_columns, display_rows = await context.display_columns_and_rows( + context.datasette, + context.database_name, + context.table_name, + context.results_description, + context.rows, + link_column=not context.is_view, + truncate_cells=context.datasette.setting("truncate_cells_html"), + sortable_columns=context.sortable_columns, + request=context.request, + ) + return { + "columns": display_columns, + "rows": display_rows, + } + + +class DisplayColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["columns"] + + +class DisplayRowsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["rows"] + + +class RenderCellExtra(Extra): + description = "Rendered HTML for each cell using the render_cell plugin hook" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + pks_for_display = ( + context.pks if context.pks else (["rowid"] if not context.is_view else []) + ) + col_names = [col[0] for col in context.results_description] + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + rendered_rows = [] + for row in context.rows: + rendered_row = {} + for value, column in zip(row, col_names): + ct = ct_map.get(column) + plugin_display_value = None + if ct: + candidate = await ct.render_cell( + value=value, + column=column, + table=context.table_name, + database=context.database_name, + datasette=context.datasette, + request=context.request, + ) + if candidate is not None: + plugin_display_value = candidate + if plugin_display_value is None: + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=context.table_name, + pks=pks_for_display, + database=context.database_name, + datasette=context.datasette, + request=context.request, + column_type=ct, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + rendered_row[column] = str(plugin_display_value) + rendered_rows.append(rendered_row) + return rendered_rows + + +class QueryExtra(Extra): + description = "Details of the underlying SQL query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "sql": context.sql, + "params": context.params, + } + + +class ColumnTypesExtra(Extra): + description = "Column type assignments for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + return { + col_name: { + "type": ct.name, + "config": ct.config, + } + for col_name, ct in ct_map.items() + } + + +class SetColumnTypeUiExtra(Extra): + description = "Column type UI metadata for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + if context.is_view: + return None + + if not await context.datasette.allowed( + action="set-column-type", + resource=TableResource( + database=context.database_name, table=context.table_name + ), + actor=context.request.actor, + ): + return None + + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(context.datasette._column_types.items()) + if context.datasette._column_type_is_applicable( + ct_cls, column_detail + ) + ], + } + return { + "path": "{}/-/set-column-type".format( + context.datasette.urls.table(context.database_name, context.table_name) + ), + "columns": columns, + } + + +class MetadataExtra(Extra): + description = "Metadata about the table and database" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + tablemetadata = await context.datasette.get_resource_metadata( + context.database_name, context.table_name + ) + + rows = await context.datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM metadata_columns + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [context.database_name, context.table_name], + ) + tablemetadata["columns"] = dict(rows) + return tablemetadata + + +class DatabaseExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.database_name + + +class TableExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.table_name + + +class DatabaseColorExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.db.color + + +class FormHiddenArgsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + form_hidden_args = [] + for key in context.request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_sort_desc", "_search", "_next") + and "__" not in key + ): + for value in context.request.args.getlist(key): + form_hidden_args.append((key, value)) + return form_hidden_args + + +class FiltersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.filters + + +class CustomTableTemplatesExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return [ + f"_table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + f"_table-table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + "_table.html", + ] + + +class SortedFacetResultsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + facet_configs = context.table_metadata.get("facets", []) + if facet_configs: + metadata_facet_names = [] + for fc in facet_configs: + if isinstance(fc, str): + metadata_facet_names.append(fc) + elif isinstance(fc, dict): + metadata_facet_names.append(list(fc.values())[0]) + metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} + metadata_facets = [] + request_facets = [] + for f in facet_results["results"].values(): + if f["name"] in metadata_order: + metadata_facets.append(f) + else: + request_facets.append(f) + metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) + request_facets.sort( + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + return metadata_facets + request_facets + else: + return sorted( + facet_results["results"].values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + + +class TableDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_table_definition(context.table_name) + + +class ViewDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_view_definition(context.table_name) + + +class RenderersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, expandable_columns, query): + renderers = {} + url_labels_extra = {} + if expandable_columns: + url_labels_extra = {"_labels": "on"} + for key, (_, can_render) in context.datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=context.datasette, + columns=context.columns or [], + rows=context.rows or [], + sql=query.get("sql", None), + query_name=None, + database=context.database_name, + table=context.table_name, + request=context.request, + view_name="table", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = context.datasette.urls.path( + path_with_format( + request=context.request, + path=context.request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + return renderers + + +class PrivateExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.private + + +class ExpandableColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + expandables = [] + db = context.datasette.databases[context.database_name] + for fk in await db.foreign_keys_for_table(context.table_name): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + +class ExtrasExtra(Extra): + description = "Available ?_extra= blocks" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + all_extras = [ + (cls.key(), cls.description) + for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + ] + return [ + { + "name": name, + "description": description, + "toggle_url": context.datasette.absolute_url( + context.request, + context.datasette.urls.path( + path_with_added_args(context.request, {"_extra": name}) + if name not in context.extras + else path_with_removed_args(context.request, {"_extra": name}) + ), + ), + "selected": name in context.extras, + } + for name, description in all_extras + ] + + +TABLE_EXTRA_BUNDLES = { + "html": [ + "suggested_facets", + "facet_results", + "facets_timed_out", + "count", + "count_sql", + "human_description_en", + "next_url", + "metadata", + "query", + "columns", + "display_columns", + "display_rows", + "database", + "table", + "database_color", + "actions", + "filters", + "renderers", + "custom_table_templates", + "sorted_facet_results", + "table_definition", + "view_definition", + "is_view", + "private", + "primary_keys", + "all_columns", + "expandable_columns", + "form_hidden_args", + "set_column_type_ui", + ] +} + + +TABLE_EXTRA_CLASSES = [ + CountExtra, + CountSqlExtra, + FacetResultsExtra, + FacetsTimedOutExtra, + SuggestedFacetsExtra, + FacetInstancesProvider, + HumanDescriptionEnExtra, + NextUrlExtra, + ColumnsExtra, + AllColumnsExtra, + PrimaryKeysExtra, + DisplayColumnsAndRowsProvider, + DisplayColumnsExtra, + DisplayRowsExtra, + RenderCellExtra, + DebugExtra, + RequestExtra, + QueryExtra, + ColumnTypesExtra, + SetColumnTypeUiExtra, + MetadataExtra, + ExtrasExtra, + DatabaseExtra, + TableExtra, + DatabaseColorExtra, + ActionsExtra, + FiltersExtra, + RenderersExtra, + CustomTableTemplatesExtra, + SortedFacetResultsExtra, + TableDefinitionExtra, + ViewDefinitionExtra, + IsViewExtra, + PrivateExtra, + ExpandableColumnsExtra, + FormHiddenArgsExtra, +] + + +table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) + + +async def resolve_table_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) From 111eeaf3702cd5ee417532beb80c746d49d92a11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:56:00 -0700 Subject: [PATCH 1831/1866] Document table JSON extras from metadata --- datasette/views/table_extras.py | 18 +++++++ docs/json_api.rst | 95 +++++++++++++++++++++++++++++++++ docs/json_api_doc.py | 20 +++++++ 3 files changed, 133 insertions(+) create mode 100644 docs/json_api_doc.py diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 2ec2adf0..e71c15d6 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -55,6 +55,7 @@ class TableExtraContext: class CountSqlExtra(Extra): + description = "SQL query used to calculate the total count" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -151,6 +152,7 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): + description = "Facet calculations that timed out" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -231,6 +233,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): + description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -258,6 +261,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): + description = "Whether this resource is a view instead of a table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -313,6 +317,7 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): + description = "Column metadata used by the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -320,6 +325,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): + description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -482,6 +488,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): + description = "Database name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +496,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): + description = "Table name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -496,6 +504,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): + description = "Color assigned to the database" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -503,6 +512,7 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): + description = "Hidden form arguments used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -519,6 +529,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): + description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -526,6 +537,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): + description = "Custom template names considered for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -537,6 +549,7 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): + description = "Facet results sorted for display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -571,6 +584,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): + description = "SQL definition for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -578,6 +592,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): + description = "SQL definition for this view" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -585,6 +600,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): + description = "Alternative output renderers available for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -619,6 +635,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): + description = "Whether this table is private to the current actor" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -626,6 +643,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): + description = "Foreign key columns that can be expanded with labels" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 65031bf4..af60a527 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,6 +235,101 @@ query string arguments: Only available if the :ref:`setting_trace_debug` setting is enabled. +.. _json_api_extra: + +Expanding table JSON responses +------------------------------ + +Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +These can be repeated or comma-separated: + +:: + + ?_extra=columns&_extra=count,next_url + +The available table extras are listed below. + +.. [[[cog + from json_api_doc import table_extras + table_extras(cog) +.. ]]] + +.. list-table:: + :header-rows: 1 + + * - Extra + - Description + * - ``count`` + - Total count of rows matching these filters (May execute additional queries.) + * - ``count_sql`` + - SQL query used to calculate the total count + * - ``facet_results`` + - Results of facets calculated against this data (May execute additional queries.) + * - ``facets_timed_out`` + - Facet calculations that timed out + * - ``suggested_facets`` + - Suggestions for facets that might return interesting results (May execute additional queries.) + * - ``human_description_en`` + - Human-readable description of the filters + * - ``next_url`` + - Full URL for the next page of results + * - ``columns`` + - Column names returned by this query + * - ``all_columns`` + - All columns in the table, regardless of _col/_nocol filtering + * - ``primary_keys`` + - Primary keys for this table + * - ``display_columns`` + - Column metadata used by the HTML table display + * - ``display_rows`` + - Row data formatted for the HTML table display + * - ``render_cell`` + - Rendered HTML for each cell using the render_cell plugin hook + * - ``debug`` + - Extra debug information + * - ``request`` + - Full information about the request + * - ``query`` + - Details of the underlying SQL query + * - ``column_types`` + - Column type assignments for this table + * - ``set_column_type_ui`` + - Column type UI metadata for this table + * - ``metadata`` + - Metadata about the table and database + * - ``extras`` + - Available ?_extra= blocks + * - ``database`` + - Database name + * - ``table`` + - Table name + * - ``database_color`` + - Color assigned to the database + * - ``actions`` + - Table or view actions made available by plugin hooks + * - ``filters`` + - Filters object used by the HTML table interface + * - ``renderers`` + - Alternative output renderers available for this table + * - ``custom_table_templates`` + - Custom template names considered for this table + * - ``sorted_facet_results`` + - Facet results sorted for display + * - ``table_definition`` + - SQL definition for this table + * - ``view_definition`` + - SQL definition for this view + * - ``is_view`` + - Whether this resource is a view instead of a table + * - ``private`` + - Whether this table is private to the current actor + * - ``expandable_columns`` + - Foreign key columns that can be expanded with labels + * - ``form_hidden_args`` + - Hidden form arguments used by the HTML table interface + +.. [[[end]]] + .. _table_arguments: Table arguments diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py new file mode 100644 index 00000000..f07c3ba7 --- /dev/null +++ b/docs/json_api_doc.py @@ -0,0 +1,20 @@ +def table_extras(cog): + from datasette.extras import ExtraScope + from datasette.views.table_extras import table_extra_registry + + cog.out("\n.. list-table::\n") + cog.out(" :header-rows: 1\n\n") + cog.out(" * - Extra\n") + cog.out(" - Description\n") + for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() + cog.out(" * - ``{}``\n".format(cls.key())) + cog.out(" - {}\n".format(description)) + cog.out("\n") From 79c8aff31df16e514616a7778fad1386ac9b4b2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:10:58 -0700 Subject: [PATCH 1832/1866] Add generated examples for table JSON extras --- datasette/extras.py | 11 + datasette/views/table_extras.py | 87 +++++- docs/json_api.rst | 470 +++++++++++++++++++++++++++----- docs/json_api_doc.py | 64 ++++- tests/test_docs.py | 10 + 5 files changed, 561 insertions(+), 81 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 786ec4f4..d867f26c 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,5 @@ import re +from dataclasses import dataclass from enum import Enum from typing import ClassVar @@ -17,6 +18,14 @@ class ExtraScope(Enum): TABLE = "table" +@dataclass(frozen=True) +class ExtraExample: + path: str | None = None + key: str | None = None + value: object | None = None + note: str | None = None + + class Provider: name: ClassVar[str | None] = None scopes: ClassVar[frozenset[ExtraScope]] = frozenset() @@ -36,6 +45,7 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None + example: ClassVar[ExtraExample | None] = None public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False @@ -52,6 +62,7 @@ class Extra(Provider): "stable": cls.stable, "expensive": cls.expensive, "docs_note": cls.docs_note, + "example": cls.example, } diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e71c15d6..0eefeaa9 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -2,7 +2,7 @@ import itertools from dataclasses import dataclass from datasette.database import QueryInterrupted -from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm from datasette.resources import TableResource from datasette.utils import ( @@ -56,6 +56,7 @@ class TableExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" + example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -64,6 +65,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" + example = ExtraExample("/fixtures/facetable.json?_extra=count") scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -121,6 +123,22 @@ class FacetInstancesProvider(Provider): class FacetResultsExtra(Extra): description = "Results of facets calculated against this data" + example = ExtraExample( + value={ + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + {"value": "CA", "label": "CA", "count": 10}, + {"value": "MI", "label": "MI", "count": 4}, + ], + } + }, + "timed_out": [], + }, + note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -153,6 +171,9 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -161,6 +182,15 @@ class FacetsTimedOutExtra(Extra): class SuggestedFacetsExtra(Extra): description = "Suggestions for facets that might return interesting results" + example = ExtraExample( + value=[ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state", + } + ], + note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -183,6 +213,9 @@ class SuggestedFacetsExtra(Extra): class HumanDescriptionEnExtra(Extra): description = "Human-readable description of the filters" + example = ExtraExample( + "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -202,6 +235,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -210,6 +244,7 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" + example = ExtraExample("/fixtures/facetable.json?_extra=columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -218,6 +253,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" + example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -226,6 +262,7 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -262,6 +299,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" + example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -318,6 +356,28 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): description = "Column metadata used by the HTML table display" + example = ExtraExample( + value=[ + { + "name": "pk", + "sortable": True, + "is_pk": True, + "type": "INTEGER", + "notnull": 0, + }, + { + "name": "created", + "sortable": True, + "is_pk": False, + "type": "TEXT", + "notnull": 0, + "description": None, + "column_type": None, + "column_type_config": None, + }, + ], + note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -334,6 +394,13 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + example = ExtraExample( + value=[ + {}, + {"content": "Custom rendered HTML"}, + ], + note="Only columns whose rendered value differs from the default are included.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -385,6 +452,7 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -396,6 +464,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" + example = ExtraExample(value={}) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -464,6 +533,7 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table and database" + example = ExtraExample("/fixtures/facetable.json?_extra=metadata") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +559,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" + example = ExtraExample("/fixtures/facetable.json?_extra=database") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -497,6 +568,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" + example = ExtraExample("/fixtures/facetable.json?_extra=table") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -505,6 +577,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + example = ExtraExample("/fixtures/facetable.json?_extra=database_color") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -513,6 +586,9 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): description = "Hidden form arguments used by the HTML table interface" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -538,6 +614,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -550,6 +627,9 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -585,6 +665,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -593,6 +674,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" + example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -601,6 +683,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=renderers") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -636,6 +719,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this table is private to the current actor" + example = ExtraExample("/fixtures/facetable.json?_extra=private") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -644,6 +728,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" + example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index af60a527..d12a388e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -254,79 +254,405 @@ The available table extras are listed below. table_extras(cog) .. ]]] -.. list-table:: - :header-rows: 1 +``count`` + Total count of rows matching these filters (May execute additional queries.) - * - Extra - - Description - * - ``count`` - - Total count of rows matching these filters (May execute additional queries.) - * - ``count_sql`` - - SQL query used to calculate the total count - * - ``facet_results`` - - Results of facets calculated against this data (May execute additional queries.) - * - ``facets_timed_out`` - - Facet calculations that timed out - * - ``suggested_facets`` - - Suggestions for facets that might return interesting results (May execute additional queries.) - * - ``human_description_en`` - - Human-readable description of the filters - * - ``next_url`` - - Full URL for the next page of results - * - ``columns`` - - Column names returned by this query - * - ``all_columns`` - - All columns in the table, regardless of _col/_nocol filtering - * - ``primary_keys`` - - Primary keys for this table - * - ``display_columns`` - - Column metadata used by the HTML table display - * - ``display_rows`` - - Row data formatted for the HTML table display - * - ``render_cell`` - - Rendered HTML for each cell using the render_cell plugin hook - * - ``debug`` - - Extra debug information - * - ``request`` - - Full information about the request - * - ``query`` - - Details of the underlying SQL query - * - ``column_types`` - - Column type assignments for this table - * - ``set_column_type_ui`` - - Column type UI metadata for this table - * - ``metadata`` - - Metadata about the table and database - * - ``extras`` - - Available ?_extra= blocks - * - ``database`` - - Database name - * - ``table`` - - Table name - * - ``database_color`` - - Color assigned to the database - * - ``actions`` - - Table or view actions made available by plugin hooks - * - ``filters`` - - Filters object used by the HTML table interface - * - ``renderers`` - - Alternative output renderers available for this table - * - ``custom_table_templates`` - - Custom template names considered for this table - * - ``sorted_facet_results`` - - Facet results sorted for display - * - ``table_definition`` - - SQL definition for this table - * - ``view_definition`` - - SQL definition for this view - * - ``is_view`` - - Whether this resource is a view instead of a table - * - ``private`` - - Whether this table is private to the current actor - * - ``expandable_columns`` - - Foreign key columns that can be expanded with labels - * - ``form_hidden_args`` - - Hidden form arguments used by the HTML table interface + ``GET /fixtures/facetable.json?_extra=count`` + + .. code-block:: json + + 15 + +``count_sql`` + SQL query used to calculate the total count + + ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` + + .. code-block:: json + + "select count(*) from facetable " + +``facet_results`` + Results of facets calculated against this data (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. + + .. code-block:: json + + { + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10 + }, + { + "value": "MI", + "label": "MI", + "count": 4 + } + ] + } + }, + "timed_out": [] + } + +``facets_timed_out`` + Facet calculations that timed out + + ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + + .. code-block:: json + + [] + +``suggested_facets`` + Suggestions for facets that might return interesting results (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. + + .. code-block:: json + + [ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state" + } + ] + +``human_description_en`` + Human-readable description of the filters + + ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en`` + + .. code-block:: json + + "where state = \"CA\" sorted by pk" + +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + +``columns`` + Column names returned by this query + + ``GET /fixtures/facetable.json?_extra=columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + + ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/facetable.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "pk" + ] + +``display_columns`` + Column metadata used by the HTML table display + + Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. + + .. code-block:: json + + [ + { + "name": "pk", + "sortable": true, + "is_pk": true, + "type": "INTEGER", + "notnull": 0 + }, + { + "name": "created", + "sortable": true, + "is_pk": false, + "type": "TEXT", + "notnull": 0, + "description": null, + "column_type": null, + "column_type_config": null + } + ] + +``display_rows`` + Row data formatted for the HTML table display + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + [ + {}, + { + "content": "Custom rendered HTML" + } + ] + +``debug`` + Extra debug information + +``request`` + Full information about the request + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/facetable.json?_size=1&_extra=query`` + + .. code-block:: json + + { + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2", + "params": {} + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``set_column_type_ui`` + Column type UI metadata for this table + +``metadata`` + Metadata about the table and database + + ``GET /fixtures/facetable.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/facetable.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/facetable.json?_extra=table`` + + .. code-block:: json + + "facetable" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/facetable.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``actions`` + Table or view actions made available by plugin hooks + +``filters`` + Filters object used by the HTML table interface + +``renderers`` + Alternative output renderers available for this table + + ``GET /fixtures/facetable.json?_extra=renderers`` + + .. code-block:: json + + { + "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on" + } + +``custom_table_templates`` + Custom template names considered for this table + + ``GET /fixtures/facetable.json?_extra=custom_table_templates`` + + .. code-block:: json + + [ + "_table-fixtures-facetable.html", + "_table-table-fixtures-facetable.html", + "_table.html" + ] + +``sorted_facet_results`` + Facet results sorted for display + + ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` + + .. code-block:: json + + [ + { + "name": "state", + "type": "column", + "hideable": true, + "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA", + "selected": false + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI", + "selected": false + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC", + "selected": false + } + ], + "truncated": false + } + ] + +``table_definition`` + SQL definition for this table + + ``GET /fixtures/facetable.json?_extra=table_definition`` + + .. code-block:: json + + "CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);" + +``view_definition`` + SQL definition for this view + + ``GET /fixtures/simple_view.json?_extra=view_definition`` + + .. code-block:: json + + "CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;" + +``is_view`` + Whether this resource is a view instead of a table + + ``GET /fixtures/simple_view.json?_extra=is_view`` + + .. code-block:: json + + true + +``private`` + Whether this table is private to the current actor + + ``GET /fixtures/facetable.json?_extra=private`` + + .. code-block:: json + + false + +``expandable_columns`` + Foreign key columns that can be expanded with labels + + ``GET /fixtures/facetable.json?_extra=expandable_columns`` + + .. code-block:: json + + [ + [ + { + "column": "_city_id", + "other_table": "facet_cities", + "other_column": "id" + }, + "name" + ] + ] + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + + ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` + + .. code-block:: json + + [ + [ + "_facet", + "state" + ], + [ + "_size", + "1" + ], + [ + "_extra", + "form_hidden_args" + ] + ] .. [[[end]]] diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index f07c3ba7..69ec6e5e 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -1,12 +1,20 @@ +import asyncio +import json +import pathlib +import tempfile +import textwrap + + def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - cog.out("\n.. list-table::\n") - cog.out(" :header-rows: 1\n\n") - cog.out(" * - Extra\n") - cog.out(" - Description\n") - for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + + live_examples = asyncio.run(_fetch_live_examples(classes)) + cog.out("\n") + for cls in classes: + example = cls.example description = cls.description or "" notes = [] if cls.expensive: @@ -15,6 +23,46 @@ def table_extras(cog): notes.append(cls.docs_note) if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(" * - ``{}``\n".format(cls.key())) - cog.out(" - {}\n".format(description)) - cog.out("\n") + + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + if example is None: + continue + + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") + + +async def _fetch_live_examples(classes): + from datasette.app import Datasette + from datasette.fixtures import write_fixture_database + + examples = {} + with tempfile.TemporaryDirectory() as tmpdir: + db_path = pathlib.Path(tmpdir) / "fixtures.db" + write_fixture_database(db_path) + datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + try: + for cls in classes: + example = cls.example + if example is None or not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] + finally: + for db in datasette.databases.values(): + if not db.is_memory: + db.close() + return examples diff --git a/tests/test_docs.py b/tests/test_docs.py index 51caf595..784755e9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -112,6 +112,16 @@ def test_table_filters_are_documented(documented_table_filters, subtests): assert f.key in documented_table_filters +def test_table_extra_examples_are_documented(): + from datasette.views.table_extras import CountExtra + + assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count" + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + assert "GET /fixtures/facetable.json?_extra=count" in section + assert ".. code-block:: json" in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 22f80b819625b9f6b5aa0661f58d97c89882a932 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:13:53 -0700 Subject: [PATCH 1833/1866] Clarify render_cell JSON extra example --- datasette/views/table_extras.py | 20 +++++++++++++++----- docs/json_api.rst | 26 +++++++++++++++++++------- tests/test_docs.py | 8 ++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 0eefeaa9..b6e653c4 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -395,11 +395,21 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" example = ExtraExample( - value=[ - {}, - {"content": "Custom rendered HTML"}, - ], - note="Only columns whose rendered value differs from the default are included.", + value={ + "rows": [ + {"id": 1, "content": "hello"}, + {"id": 4, "content": "RENDER_CELL_DEMO"}, + ], + "render_cell": [ + {}, + {"content": "Custom rendered HTML"}, + ], + }, + note=( + "The ``render_cell`` array has one item per row, in the same order as " + "the ``rows`` array. Each object is keyed by column name. Only columns " + "whose rendered value differs from the default are included." + ), ) scopes = frozenset({ExtraScope.TABLE}) diff --git a/docs/json_api.rst b/docs/json_api.rst index d12a388e..24d59577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -428,16 +428,28 @@ The available table extras are listed below. ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook - Only columns whose rendered value differs from the default are included. + The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. .. code-block:: json - [ - {}, - { - "content": "Custom rendered HTML" - } - ] + { + "rows": [ + { + "id": 1, + "content": "hello" + }, + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + {}, + { + "content": "Custom rendered HTML" + } + ] + } ``debug`` Extra debug information diff --git a/tests/test_docs.py b/tests/test_docs.py index 784755e9..c4e0a849 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -122,6 +122,14 @@ def test_table_extra_examples_are_documented(): assert ".. code-block:: json" in section +def test_render_cell_extra_example_explains_row_and_column_mapping(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("``render_cell``")[-1].split("``query``")[0] + assert "same order as the ``rows`` array" in section + assert '"rows": [' in section + assert '"render_cell": [' in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 0fa872d43842d87af9e7b8c193f90addfcf164ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:20:06 -0700 Subject: [PATCH 1834/1866] Add debug and request JSON extra examples --- datasette/views/table_extras.py | 2 ++ docs/json_api.rst | 31 +++++++++++++++++++++++++++++++ tests/test_docs.py | 13 +++++++++++++ 3 files changed, 46 insertions(+) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index b6e653c4..e888ee9f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -308,6 +308,7 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + example = ExtraExample("/fixtures/facetable.json?_extra=debug") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -321,6 +322,7 @@ class DebugExtra(Extra): class RequestExtra(Extra): description = "Full information about the request" + example = ExtraExample("/fixtures/facetable.json?_extra=request") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 24d59577..d418d16c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,9 +454,40 @@ The available table extras are listed below. ``debug`` Extra debug information + ``GET /fixtures/facetable.json?_extra=debug`` + + .. code-block:: json + + { + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", + "url_vars": { + "database": "fixtures", + "table": "facetable", + "format": "json" + }, + "nofacet": null, + "nosuggest": null + } + ``request`` Full information about the request + ``GET /fixtures/facetable.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/facetable.json?_extra=request", + "path": "/fixtures/facetable.json", + "full_path": "/fixtures/facetable.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + ``query`` Details of the underlying SQL query diff --git a/tests/test_docs.py b/tests/test_docs.py index c4e0a849..3aa67730 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -130,6 +130,19 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): assert '"render_cell": [' in section +def test_debug_and_request_extra_examples_are_documented(): + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + + debug_section = section.split("``debug``")[-1].split("``request``")[0] + assert "GET /fixtures/facetable.json?_extra=debug" in debug_section + assert '"url_vars": {' in debug_section + + request_section = section.split("``request``")[-1].split("``query``")[0] + assert "GET /fixtures/facetable.json?_extra=request" in request_section + assert '"full_path":' in request_section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 4d6daa175a67c4c6e895fe8b32ae051b1e9136a7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 02:56:27 -0700 Subject: [PATCH 1835/1866] Add row and query JSON extras --- datasette/extras.py | 7 + datasette/views/database.py | 46 +++- datasette/views/row.py | 67 ++---- datasette/views/table_extras.py | 258 ++++++++++++++++++--- docs/json_api.rst | 386 +++++++++++++++++++++++++++++++- docs/json_api_doc.py | 141 ++++++++---- tests/test_api.py | 22 ++ tests/test_docs.py | 16 +- tests/test_table_api.py | 49 ++++ 9 files changed, 862 insertions(+), 130 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d867f26c..f655e517 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -16,6 +16,8 @@ def extra_names_from_request(request): class ExtraScope(Enum): TABLE = "table" + ROW = "row" + QUERY = "query" @dataclass(frozen=True) @@ -46,11 +48,16 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None example: ClassVar[ExtraExample | None] = None + examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None + @classmethod + def example_for_scope(cls, scope): + return cls.examples.get(scope, cls.example) + @classmethod def documentation(cls): return { diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..96a58758 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ import sqlite_utils import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict @@ -38,6 +39,11 @@ from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_extras import ( + QueryExtraContext, + resolve_query_extras, + table_extra_registry, +) from . import Context @@ -692,6 +698,34 @@ class QueryView(View): except DatasetteError: raise + extras = extra_names_from_request(request) + metadata = None + data = {"ok": True, "rows": rows, "columns": columns} + if extras: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=params_for_query, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=metadata, + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) + # Handle formats from plugins if format_ == "csv": if not sql: @@ -721,7 +755,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"ok": True, "rows": rows, "columns": columns}, + data=data, ) if asyncio.iscoroutine(result): result = await result @@ -770,11 +804,11 @@ class QueryView(View): ) } ) - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) - + if metadata is None: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( diff --git a/datasette/views/row.py b/datasette/views/row.py index 4eacfe49..3fe213d7 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -15,6 +15,7 @@ import json import markupsafe import sqlite_utils from .table import display_columns_and_rows, _get_extras +from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry class RowView(DataView): @@ -172,52 +173,26 @@ class RowView(DataView): extras.add("foreign_key_tables") # Process extras - if "foreign_key_tables" in extras: - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - if "render_cell" in extras: - # Call render_cell plugin hook for each cell - ct_map = await self.ds.get_column_types(database, table) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, columns): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table, - pks=resolved.pks, - database=database, - datasette=self.ds, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - data["render_cell"] = rendered_rows + row_extra_context = RowExtraContext( + datasette=self.ds, + request=request, + resolved=resolved, + db=db, + database_name=database, + table_name=table, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + pks=pks, + pk_values=pk_values, + sql=resolved.sql, + params=resolved.params, + extras=extras, + extra_registry=table_extra_registry, + foreign_key_tables=self.foreign_key_tables, + ) + data.update(await resolve_row_extras(extras, row_extra_context)) return ( data, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e888ee9f..ec104be3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,6 +8,7 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, + named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -52,6 +53,50 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + scope: ExtraScope = ExtraScope.TABLE + + +@dataclass(frozen=True) +class RowExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + private: bool + rows: list + columns: list + results_description: list + pks: list + pk_values: list + sql: str + params: dict + extras: set + extra_registry: ExtraRegistry + foreign_key_tables: object + scope: ExtraScope = ExtraScope.ROW + + +@dataclass(frozen=True) +class QueryExtraContext: + datasette: object + request: object + db: object + database_name: str + private: bool + rows: list + columns: list + sql: str | None + params: dict + query_name: str | None + stored_query: object + stored_query_write: bool + error: str | None + metadata: dict + extras: set + extra_registry: ExtraRegistry + scope: ExtraScope = ExtraScope.QUERY class CountSqlExtra(Extra): @@ -245,7 +290,15 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" example = ExtraExample("/fixtures/facetable.json?_extra=columns") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=columns" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.columns @@ -263,7 +316,12 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=primary_keys" + ) + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.pks @@ -309,21 +367,49 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" example = ExtraExample("/fixtures/facetable.json?_extra=debug") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=debug" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - return { - "resolved": repr(context.resolved), + debug = { "url_vars": context.request.url_vars, - "nofacet": context.nofacet, - "nosuggest": context.nosuggest, } + if context.scope == ExtraScope.TABLE: + debug["resolved"] = repr(context.resolved) + elif context.scope == ExtraScope.ROW: + debug["resolved"] = { + "table": context.table_name, + "sql": context.sql, + "params": context.params, + "pks": context.pks, + "pk_values": context.pk_values, + } + if hasattr(context, "nofacet"): + debug["nofacet"] = context.nofacet + if hasattr(context, "nosuggest"): + debug["nosuggest"] = context.nosuggest + return debug class RequestExtra(Extra): description = "Full information about the request" example = ExtraExample("/fixtures/facetable.json?_extra=request") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=request" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return { @@ -413,15 +499,48 @@ class RenderCellExtra(Extra): "whose rendered value differs from the default are included." ), ) - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + value={ + "rows": [{"id": 4, "content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item for the requested row. " + "The object is keyed by column name. Only columns whose rendered " + "value differs from the default are included." + ), + ), + ExtraScope.QUERY: ExtraExample( + value={ + "rows": [{"content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item per query result row, in " + "the same order as the ``rows`` array. Each object is keyed by " + "column name. Only columns whose rendered value differs from the " + "default are included." + ), + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + table_name = getattr(context, "table_name", None) + is_view = getattr(context, "is_view", False) + pks = getattr(context, "pks", []) pks_for_display = ( - context.pks if context.pks else (["rowid"] if not context.is_view else []) + pks if pks else (["rowid"] if table_name and not is_view else []) ) - col_names = [col[0] for col in context.results_description] - ct_map = await context.datasette.get_column_types( - context.database_name, context.table_name + if hasattr(context, "results_description"): + col_names = [col[0] for col in context.results_description] + else: + col_names = context.columns + ct_map = ( + await context.datasette.get_column_types(context.database_name, table_name) + if table_name + else {} ) rendered_rows = [] for row in context.rows: @@ -433,7 +552,7 @@ class RenderCellExtra(Extra): candidate = await ct.render_cell( value=value, column=column, - table=context.table_name, + table=table_name, database=context.database_name, datasette=context.datasette, request=context.request, @@ -445,7 +564,7 @@ class RenderCellExtra(Extra): row=row, value=value, column=column, - table=context.table_name, + table=table_name, pks=pks_for_display, database=context.database_name, datasette=context.datasette, @@ -465,19 +584,36 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=query" + ), + ExtraScope.QUERY: [ + ExtraExample("/fixtures/-/query.json?sql=select+1+as+one&_extra=query"), + ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), + ], + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + params = context.params + if context.scope == ExtraScope.QUERY and context.sql: + parameter_names = set(derive_named_parameters(context.sql)) + params = { + key: value + for key, value in dict(context.params).items() + if key in parameter_names + } return { "sql": context.sql, - "params": context.params, + "params": params, } class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -544,11 +680,22 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): - description = "Metadata about the table and database" + description = "Metadata about the table, database or stored query" example = ExtraExample("/fixtures/facetable.json?_extra=metadata") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=metadata" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + if context.scope == ExtraScope.QUERY: + return context.metadata + tablemetadata = await context.datasette.get_resource_metadata( context.database_name, context.table_name ) @@ -572,7 +719,15 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" example = ExtraExample("/fixtures/facetable.json?_extra=database") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.database_name @@ -581,7 +736,10 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" example = ExtraExample("/fixtures/facetable.json?_extra=table") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.table_name @@ -590,7 +748,15 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" example = ExtraExample("/fixtures/facetable.json?_extra=database_color") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database_color" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.db.color @@ -703,6 +869,8 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} + table_name = getattr(context, "table_name", None) + view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( can_render, @@ -710,11 +878,11 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=None, + query_name=getattr(context, "query_name", None), database=context.database_name, - table=context.table_name, + table=table_name, request=context.request, - view_name="table", + view_name=view_name, ) it_can_render = await await_me_maybe(it_can_render) if it_can_render: @@ -730,9 +898,17 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): - description = "Whether this table is private to the current actor" + description = "Whether this resource is private to the current actor" example = ExtraExample("/fixtures/facetable.json?_extra=private") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=private" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.private @@ -752,14 +928,27 @@ class ExpandableColumnsExtra(Extra): return expandables +class ForeignKeyTablesExtra(Extra): + description = "Tables that link to this row using foreign keys" + example = ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + ) + scopes = frozenset({ExtraScope.ROW}) + + async def resolve(self, context): + return await context.foreign_key_tables( + context.database_name, context.table_name, context.pk_values + ) + + class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): all_extras = [ (cls.key(), cls.description) - for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + for cls in context.extra_registry.public_classes_for_scope(context.scope) ] return [ { @@ -850,6 +1039,7 @@ TABLE_EXTRA_CLASSES = [ IsViewExtra, PrivateExtra, ExpandableColumnsExtra, + ForeignKeyTablesExtra, FormHiddenArgsExtra, ] @@ -859,3 +1049,11 @@ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) async def resolve_table_extras(extras, context): return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) + + +async def resolve_row_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.ROW) + + +async def resolve_query_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.QUERY) diff --git a/docs/json_api.rst b/docs/json_api.rst index d418d16c..379d26a0 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -237,23 +237,26 @@ query string arguments: .. _json_api_extra: -Expanding table JSON responses ------------------------------- +Expanding JSON responses +------------------------ -Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters. These can be repeated or comma-separated: :: ?_extra=columns&_extra=count,next_url -The available table extras are listed below. - .. [[[cog from json_api_doc import table_extras table_extras(cog) .. ]]] +Table JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The available table extras are listed below. + ``count`` Total count of rows matching these filters (May execute additional queries.) @@ -459,12 +462,12 @@ The available table extras are listed below. .. code-block:: json { - "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "url_vars": { "database": "fixtures", "table": "facetable", "format": "json" }, + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "nofacet": null, "nosuggest": null } @@ -511,7 +514,7 @@ The available table extras are listed below. Column type UI metadata for this table ``metadata`` - Metadata about the table and database + Metadata about the table, database or stored query ``GET /fixtures/facetable.json?_extra=metadata`` @@ -649,7 +652,7 @@ The available table extras are listed below. true ``private`` - Whether this table is private to the current actor + Whether this resource is private to the current actor ``GET /fixtures/facetable.json?_extra=private`` @@ -697,6 +700,373 @@ The available table extras are listed below. ] ] +Row JSON responses +~~~~~~~~~~~~~~~~~~ + +The following extras are available for row JSON responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` + + .. code-block:: json + + [ + "id", + "content" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "id" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json" + }, + "resolved": { + "table": "simple_primary_key", + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + }, + "pks": [ + "id" + ], + "pk_values": [ + "1" + ] + } + } + +``request`` + Full information about the request + + ``GET /fixtures/simple_primary_key/1.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request", + "path": "/fixtures/simple_primary_key/1.json", + "full_path": "/fixtures/simple_primary_key/1.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/simple_primary_key/1.json?_extra=query`` + + .. code-block:: json + + { + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + } + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/simple_primary_key/1.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/simple_primary_key/1.json?_extra=table`` + + .. code-block:: json + + "simple_primary_key" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/simple_primary_key/1.json?_extra=private`` + + .. code-block:: json + + false + +``foreign_key_tables`` + Tables that link to this row using foreign keys + + ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + + .. code-block:: json + + [ + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_label", + "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1" + } + ] + +Query JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The following extras are available for arbitrary SQL query responses and stored, named query responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` + + .. code-block:: json + + [ + "one" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "format": "json" + } + } + +``request`` + Full information about the request + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "path": "/fixtures/-/query.json", + "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "host": "localhost", + "args": { + "sql": [ + "select 1 as one" + ], + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` + + .. code-block:: json + + { + "sql": "select 1 as one", + "params": {} + } + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=query`` + + .. code-block:: json + + { + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "params": { + "text": "town" + } + } + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + + .. code-block:: json + + { + "database": "fixtures", + "name": "neighborhood_search", + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "title": "Search neighborhoods", + "description": null, + "description_html": null, + "hide_sql": false, + "fragment": null, + "params": [], + "parameters": [], + "is_write": false, + "is_private": false, + "is_trusted": true, + "owner_id": null, + "on_success_message": null, + "on_success_message_sql": null, + "on_success_redirect": null, + "on_error_message": null, + "on_error_redirect": null + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database`` + + .. code-block:: json + + "fixtures" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` + + .. code-block:: json + + false + .. [[[end]]] .. _table_arguments: diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 69ec6e5e..44ef4a42 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -9,39 +9,80 @@ def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + scopes = [ + ( + ExtraScope.TABLE, + "Table JSON responses", + "The available table extras are listed below.", + ), + ( + ExtraScope.ROW, + "Row JSON responses", + "The following extras are available for row JSON responses.", + ), + ( + ExtraScope.QUERY, + "Query JSON responses", + ( + "The following extras are available for arbitrary SQL query " + "responses and stored, named query responses." + ), + ), + ] + classes_by_scope = [ + (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope)) + for scope, heading, intro in scopes + ] - live_examples = asyncio.run(_fetch_live_examples(classes)) + live_examples = asyncio.run( + _fetch_live_examples( + [ + (scope, cls) + for scope, _, _, classes in classes_by_scope + for cls in classes + ] + ) + ) cog.out("\n") - for cls in classes: - example = cls.example - description = cls.description or "" - notes = [] - if cls.expensive: - notes.append("May execute additional queries.") - if cls.docs_note: - notes.append(cls.docs_note) - if notes: - description = "{} ({})".format(description, " ".join(notes)).strip() + for scope, heading, intro, classes in classes_by_scope: + cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) + cog.out("{}\n\n".format(intro)) + for cls in classes: + examples = _examples_for_scope(cls, scope) + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out("``{}``\n".format(cls.key())) - cog.out(" {}\n\n".format(description)) - if example is None: - continue - - if example.path: - value = live_examples[(example.path, example.key or cls.key())] - cog.out(" ``GET {}``\n\n".format(example.path)) - else: - value = example.value - if example.note: - cog.out(" {}\n\n".format(example.note)) - cog.out(" .. code-block:: json\n\n") - cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) - cog.out("\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + for example in examples: + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") -async def _fetch_live_examples(classes): +def _examples_for_scope(cls, scope): + examples = cls.example_for_scope(scope) + if examples is None: + return [] + if isinstance(examples, list): + return examples + return [examples] + + +async def _fetch_live_examples(scoped_classes): from datasette.app import Datasette from datasette.fixtures import write_fixture_database @@ -49,18 +90,40 @@ async def _fetch_live_examples(classes): with tempfile.TemporaryDirectory() as tmpdir: db_path = pathlib.Path(tmpdir) / "fixtures.db" write_fixture_database(db_path) - datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + datasette = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": textwrap.dedent(""" + select _neighborhood, facet_cities.name, state + from facetable + join facet_cities + on facetable._city_id = facet_cities.id + where _neighborhood like '%' || :text || '%' + order by _neighborhood; + """), + "title": "Search neighborhoods", + } + } + } + } + }, + ) try: - for cls in classes: - example = cls.example - if example is None or not example.path: - continue - key = example.key or cls.key() - response = await datasette.client.get(example.path) - assert response.status_code == 200, example.path - data = response.json() - assert key in data, "{} missing from {}".format(key, example.path) - examples[(example.path, key)] = data[key] + for scope, cls in scoped_classes: + for example in _examples_for_scope(cls, scope): + if not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): if not db.is_memory: diff --git a/tests/test_api.py b/tests/test_api.py index f6187529..e1385b6f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_extras(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "fixtures" + assert data["table"] == "simple_primary_key" + assert data["primary_keys"] == ["id"] + assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0' + assert data["query"]["params"] == {"p0": "1"} + assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json", + } + assert len(data["foreign_key_tables"]) == 5 + + @pytest.mark.asyncio async def test_row_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages""" diff --git a/tests/test_docs.py b/tests/test_docs.py index 3aa67730..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -132,7 +132,7 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): def test_debug_and_request_extra_examples_are_documented(): content = (docs_path / "json_api.rst").read_text() - section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0] debug_section = section.split("``debug``")[-1].split("``request``")[0] assert "GET /fixtures/facetable.json?_extra=debug" in debug_section @@ -143,6 +143,20 @@ def test_debug_and_request_extra_examples_are_documented(): assert '"full_path":' in request_section +def test_row_and_query_extra_sections_are_documented(): + content = (docs_path / "json_api.rst").read_text() + assert "Row JSON responses" in content + assert ( + "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``" + in content + ) + assert "Query JSON responses" in content + assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content + assert ( + "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content + ) + + @pytest.fixture(scope="session") def documented_labels(): labels = set() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index eeb3dc8b..388e3979 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -68,6 +68,55 @@ async def test_table_shape_arrayfirst(ds_client): ] +@pytest.mark.asyncio +async def test_query_extras_for_arbitrary_sql(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?" + + urllib.parse.urlencode( + { + "sql": "select 1 as one", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["rows"] == [{"one": 1}] + assert data["columns"] == ["one"] + assert data["database"] == "fixtures" + assert data["query"]["sql"] == "select 1 as one" + assert data["request"]["path"] == "/fixtures/-/query.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "format": "json", + } + + +@pytest.mark.asyncio +async def test_query_extras_for_stored_query(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?" + + urllib.parse.urlencode( + { + "text": "town", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["_neighborhood", "name", "state"] + assert data["database"] == "fixtures" + assert data["query"]["sql"].strip().startswith("select _neighborhood") + assert data["query"]["params"]["text"] == "town" + assert data["request"]["path"] == "/fixtures/neighborhood_search.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "neighborhood_search", + "format": "json", + } + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From d8605ef4c2c054610d2f4fbf1c00d182afa617e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 19:58:00 -0700 Subject: [PATCH 1836/1866] Fix execute_isolated_fn() against immutable databases execute_isolated_fn() always opened its temporary connection with write=True, which is not allowed for immutable databases - so APIs that rely on it, like SQL analysis when storing a query, failed. An immutable database can never receive writes, so there is no write queue to block: in that case the function now opens a read-only connection and runs it on the executor, bypassing the write thread entirely. Mutable databases keep the existing write-thread behavior. Also fixed a latent bug in the write thread where a connect() failure for an isolated task would crash the thread instead of delivering the exception back to the caller. Closes #2768 Co-Authored-By: Claude Fable 5 --- datasette/database.py | 46 +++++++++++++++++++------------- tests/test_internals_database.py | 33 +++++++++++++++++++++++ tests/test_queries.py | 34 ++++++++++++++++++++++- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 0a32442c..6cd5d11e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -298,13 +298,14 @@ class Database: async def execute_isolated_fn(self, fn): self._check_not_closed() - # Open a new connection just for the duration of this function + # Open a new connection just for the duration of this function, # blocking the write queue to avoid any writes occurring during it - if self.ds.executor is None: - # non-threaded mode - isolated_connection = self.connect(write=True) + write = self.is_mutable + + def _run(): + isolated_connection = self.connect(write=write) try: - result = fn(isolated_connection) + return fn(isolated_connection) finally: isolated_connection.close() try: @@ -312,10 +313,18 @@ class Database: except ValueError: # Was probably a memory connection pass - return result - else: - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is no + # write queue to block; run against a fresh read-only connection + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -449,20 +458,21 @@ class Database: if conn_exception is not None: exception = conn_exception elif task.isolated_connection: - isolated_connection = self.connect(write=True) try: - result = task.fn(isolated_connection) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass else: try: if task.transaction: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index bb209649..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -863,6 +863,39 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_execute_isolated_connect_failure_does_not_kill_write_thread(): + # A connect() failure for an isolated task should be returned to the + # caller as an exception, not crash the write thread + class ConnectError(Exception): + pass + + ds = Datasette(memory=True) + db = ds.add_memory_database("test_isolated_connect_failure") + # Start the write thread with a healthy dedicated write connection + await db.execute_write("create table dogs (id integer primary key)") + + original_connect = db.connect + + def broken_connect(write=False): + raise ConnectError("Could not connect") + + db.connect = broken_connect + try: + with pytest.raises(ConnectError): + await asyncio.wait_for(db.execute_isolated_fn(lambda conn: None), timeout=2) + finally: + db.connect = original_connect + + # Write thread should still be alive and processing tasks + assert db._write_thread.is_alive() + await db.execute_write("insert into dogs (id) values (1)") + count = await db.execute_isolated_fn( + lambda conn: conn.execute("select count(*) from dogs").fetchone()[0] + ) + assert count == 1 + + @pytest.mark.asyncio async def test_analyze_sql(): ds = Datasette(memory=True) diff --git a/tests/test_queries.py b/tests/test_queries.py index 6e9bcbdb..0354f73a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -9,7 +9,7 @@ from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden -from datasette.utils.sqlite import supports_returning +from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -593,6 +593,38 @@ async def test_query_store_api_creates_read_only_query(): assert data["query"]["owner_id"] == "root" +@pytest.mark.asyncio +async def test_query_store_api_creates_query_for_immutable_database(tmp_path): + db_path = tmp_path / "immutable.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("create table dogs (id integer primary key, name text)") + conn.commit() + conn.close() + + ds = Datasette([], immutables=[str(db_path)], default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.post( + "/immutable/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + } + }, + ) + + ds.close() + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + + @pytest.mark.asyncio async def test_query_list_and_definition_api(): ds = Datasette(memory=True) From 3c1012dcc2995d184ea24fe70e8ccd6580592aff Mon Sep 17 00:00:00 2001 From: Viraat Das Date: Wed, 10 Jun 2026 20:15:03 -0700 Subject: [PATCH 1837/1866] Fix write query failing when a named parameter is called :sql (#2765) Closes #2761 --- .../templates/_sql_parameter_scripts.html | 34 ++++++++++++----- datasette/templates/_sql_parameters.html | 5 ++- datasette/views/execute_write.py | 2 + datasette/views/query_helpers.py | 14 ++++--- tests/test_api_write.py | 38 +++++++++++++++++++ tests/test_html.py | 4 +- tests/test_stored_queries.py | 2 +- 7 files changed, 79 insertions(+), 20 deletions(-) diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 159a141c..9b83889e 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -27,16 +27,20 @@ window.datasetteSqlParameters = (() => { manager.section .querySelectorAll("[data-parameter-control]") .forEach((control) => { - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); } - function createControl(parameter, id, state) { + function createControl(parameter, id, state, namePrefix) { const control = document.createElement(state.expanded ? "textarea" : "input"); control.id = id; - control.name = parameter; + control.name = `${namePrefix || ""}${parameter}`; control.value = state.value; control.setAttribute("data-parameter-control", ""); + control.dataset.parameterName = parameter; if (state.expanded) { control.rows = 5; } else { @@ -53,10 +57,16 @@ window.datasetteSqlParameters = (() => { value, selectionStart ) { - const replacement = createControl(control.name, control.id, { - value: value === undefined ? control.value : value, - expanded: expand, - }); + const parameter = control.dataset.parameterName; + const replacement = createControl( + parameter, + control.id, + { + value: value === undefined ? control.value : value, + expanded: expand, + }, + manager.namePrefix + ); button.textContent = expand ? "Collapse" : "Expand"; button.setAttribute("aria-expanded", expand ? "true" : "false"); control.replaceWith(replacement); @@ -64,7 +74,7 @@ window.datasetteSqlParameters = (() => { if (selectionStart !== undefined && replacement.setSelectionRange) { replacement.setSelectionRange(selectionStart, selectionStart); } - manager.parameterState.set(replacement.name, controlState(replacement)); + manager.parameterState.set(parameter, controlState(replacement)); } function renderParameters(manager, parameters) { @@ -99,7 +109,7 @@ window.datasetteSqlParameters = (() => { label.htmlFor = id; label.textContent = parameter; - const control = createControl(parameter, id, state); + const control = createControl(parameter, id, state, manager.namePrefix); row.append(label, control); if (manager.allowExpand) { @@ -124,7 +134,10 @@ window.datasetteSqlParameters = (() => { if (!control.matches || !control.matches("[data-parameter-control]")) { return; } - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); if (!manager.allowExpand) { @@ -230,6 +243,7 @@ window.datasetteSqlParameters = (() => { ? section.dataset.allowExpand === "1" : false : options.allowExpand, + namePrefix: section ? section.dataset.parameterNamePrefix || "" : "", parameterState: new Map(), }; if (section) { diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html index 58801d40..b5c1bde8 100644 --- a/datasette/templates/_sql_parameters.html +++ b/datasette/templates/_sql_parameters.html @@ -1,9 +1,10 @@ -
    +{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} +
    {% if parameter_names %}

    Parameters

    {% for parameter in parameter_names %} {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

    {% if sql_parameters_allow_expand|default(false) %} {% endif %}

    +

    {% if sql_parameters_allow_expand|default(false) %} {% endif %}

    {% endfor %} {% endif %}
    diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index c5d55b80..2817f56e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -9,6 +9,7 @@ from .base import BaseView, _error from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -295,6 +296,7 @@ class ExecuteWriteView(BaseView): "execute_write_columns": execute_write_columns, "execute_write_display_rows": execute_write_display_rows, "execute_write_truncated": execute_write_truncated, + "sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX, "execute_disabled": bool(execute_disabled_reason), "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 9efe3f81..026a999f 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -49,6 +49,8 @@ _query_write_fields = { "on_error_redirect", } +SQL_PARAMETER_FORM_PREFIX = "_sql_param_" + class QueryValidationError(Exception): def __init__(self, message, status=400, *, flash=False): @@ -289,11 +291,13 @@ def _coerce_execute_write_payload(data, is_json): ) params = data.get("params") or {} else: - params = { - key: value - for key, value in data.items() - if key not in {"sql", "csrftoken", "_json"} - } + params = {} + for key, value in data.items(): + if key in {"sql", "csrftoken", "_json"}: + continue + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") return data.get("sql"), params diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 64f91701..b7ceb6b2 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,44 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_execute_write_form_parameter_called_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_parameter_sql", name="data") + await db.execute_write("create table docs (id integer primary key, title text)") + await db.execute_write("insert into docs (id, title) values (1, 'Initial')") + await ds.invoke_startup() + + form_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + params={"sql": "update docs set title = :sql where id = :id"}, + ) + assert form_response.status_code == 200 + assert 'data-parameter-name-prefix="_sql_param_"' in form_response.text + assert '' in form_response.text + assert 'name="_sql_param_sql"' in form_response.text + assert 'data-parameter-name="sql"' in form_response.text + assert 'name="_sql_param_id"' in form_response.text + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update docs set title = :sql where id = :id", + "_sql_param_sql": "Updated", + "_sql_param_id": "1", + }, + ) + + assert response.status_code == 200 + assert "Query executed, 1 row affected" in response.text + assert (await db.execute("select title from docs where id = 1")).first()[ + 0 + ] == "Updated" + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", diff --git a/tests/test_html.py b/tests/test_html.py index bb7f612e..20ab22bc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -342,7 +342,7 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text @@ -351,7 +351,7 @@ async def test_query_parameter_form_fields(ds_client): response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py index 2c648d5f..46420749 100644 --- a/tests/test_stored_queries.py +++ b/tests/test_stored_queries.py @@ -201,7 +201,7 @@ def test_error_in_on_success_message_sql(stored_write_client): def test_custom_params(stored_write_client): response = stored_write_client.get("/data/update_name?extra=foo") assert ( - '' + '' in response.text ) From f4b450603559b6a6412ed67e9eb170255dd1ab6b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:49:23 -0700 Subject: [PATCH 1838/1866] Remove legacy ?_extras= row parameter The pre-1.0 ?_extras= (plural) parameter was kept for backwards compatibility with the old row JSON API. ?_extra= is the documented mechanism now that row pages share the extras registry. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 ----- tests/test_api.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3fe213d7..ce15a822 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -165,13 +165,8 @@ class RowView(DataView): "primary_key_values": pk_values, } - # Handle _extra parameter (new style) extras = _get_extras(request) - # Also support legacy _extras parameter for backward compatibility - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - extras.add("foreign_key_tables") - # Process extras row_extra_context = RowExtraContext( datasette=self.ds, diff --git a/tests/test_api.py b/tests/test_api.py index e1385b6f..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client): @pytest.mark.asyncio async def test_row_foreign_key_tables(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) assert response.status_code == 200 # Foreign keys are sorted by (other_table, column, other_column) From d825d8c4f38d980356abc50c739a440585253062 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:53:41 -0700 Subject: [PATCH 1839/1866] Remove _get_extras() shim in favor of extra_names_from_request() Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 +++-- datasette/views/table.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce15a822..c6721ca0 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -14,7 +14,8 @@ from datasette.plugins import pm import json import markupsafe import sqlite_utils -from .table import display_columns_and_rows, _get_extras +from datasette.extras import extra_names_from_request +from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -165,7 +166,7 @@ class RowView(DataView): "primary_key_values": pk_values, } - extras = _get_extras(request) + extras = extra_names_from_request(request) # Process extras row_extra_context = RowExtraContext( diff --git a/datasette/views/table.py b/datasette/views/table.py index c2d520f8..1b298c50 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -854,10 +854,6 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) -def _get_extras(request): - return extra_names_from_request(request) - - async def _columns_to_select(table_columns, pks, request): columns = list(table_columns) if "_col" in request.args: @@ -1461,7 +1457,7 @@ async def table_view_data( rows = rows[:page_size] # Resolve extras - extras = _get_extras(request) + extras = extra_names_from_request(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": From df8a61450b478e66c458b3f05c286daeb2c2a6b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:57:02 -0700 Subject: [PATCH 1840/1866] Remove hasattr/getattr probing from multi-scope extras TableExtraContext, RowExtraContext and QueryExtraContext now share normalized table_name, is_view, pks and query_name fields (defaulting to None/False where inapplicable) so DebugExtra, RenderCellExtra and RenderersExtra can read them directly. RenderCellExtra uses context.columns in every scope - the table and row views both derive columns from results.description so output is unchanged. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 1 - datasette/views/table_extras.py | 30 +++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index c6721ca0..e15dfce9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -179,7 +179,6 @@ class RowView(DataView): private=private, rows=rows, columns=columns, - results_description=results.description, pks=pks, pk_values=pk_values, sql=resolved.sql, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ec104be3..63c87a6f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -53,6 +53,7 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + query_name: str | None = None scope: ExtraScope = ExtraScope.TABLE @@ -67,7 +68,6 @@ class RowExtraContext: private: bool rows: list columns: list - results_description: list pks: list pk_values: list sql: str @@ -75,6 +75,7 @@ class RowExtraContext: extras: set extra_registry: ExtraRegistry foreign_key_tables: object + is_view: bool = False scope: ExtraScope = ExtraScope.ROW @@ -96,6 +97,9 @@ class QueryExtraContext: metadata: dict extras: set extra_registry: ExtraRegistry + table_name: str | None = None + is_view: bool = False + pks: list | None = None scope: ExtraScope = ExtraScope.QUERY @@ -383,6 +387,8 @@ class DebugExtra(Extra): } if context.scope == ExtraScope.TABLE: debug["resolved"] = repr(context.resolved) + debug["nofacet"] = context.nofacet + debug["nosuggest"] = context.nosuggest elif context.scope == ExtraScope.ROW: debug["resolved"] = { "table": context.table_name, @@ -391,10 +397,6 @@ class DebugExtra(Extra): "pks": context.pks, "pk_values": context.pk_values, } - if hasattr(context, "nofacet"): - debug["nofacet"] = context.nofacet - if hasattr(context, "nosuggest"): - debug["nosuggest"] = context.nosuggest return debug @@ -527,16 +529,10 @@ class RenderCellExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - table_name = getattr(context, "table_name", None) - is_view = getattr(context, "is_view", False) - pks = getattr(context, "pks", []) - pks_for_display = ( - pks if pks else (["rowid"] if table_name and not is_view else []) + table_name = context.table_name + pks_for_display = context.pks or ( + ["rowid"] if table_name and not context.is_view else [] ) - if hasattr(context, "results_description"): - col_names = [col[0] for col in context.results_description] - else: - col_names = context.columns ct_map = ( await context.datasette.get_column_types(context.database_name, table_name) if table_name @@ -545,7 +541,7 @@ class RenderCellExtra(Extra): rendered_rows = [] for row in context.rows: rendered_row = {} - for value, column in zip(row, col_names): + for value, column in zip(row, context.columns): ct = ct_map.get(column) plugin_display_value = None if ct: @@ -869,7 +865,7 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} - table_name = getattr(context, "table_name", None) + table_name = context.table_name view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( @@ -878,7 +874,7 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=getattr(context, "query_name", None), + query_name=context.query_name, database=context.database_name, table=table_name, request=context.request, From ab62ec96d187fa05f7d672d48a0b3f962fb8c228 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:45:13 -0700 Subject: [PATCH 1841/1866] Fix _extra=private for arbitrary SQL query pages QueryView hardcoded private=False unless the request was for a stored query, so /db/-/query.json?_extra=private reported false even when execute-sql was restricted to the authenticated actor. Use check_visibility() like the table and row views do. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 6 ++++-- tests/test_table_api.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 96a58758..e6efddea 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -612,11 +612,13 @@ class QueryView(View): ) else: - await datasette.ensure_permission( + visible, private = await datasette.check_visibility( + request.actor, action="execute-sql", resource=DatabaseResource(database=database), - actor=request.actor, ) + if not visible: + raise Forbidden("execute-sql") # Flattened because of ?sql=&name1=value1&name2=value2 feature params = {key: request.args.get(key) for key in request.args} diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 388e3979..4ab2f596 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,29 @@ async def test_query_extras_for_stored_query(ds_client): } +def test_query_extra_private_for_arbitrary_sql(): + with make_app_client(config={"allow_sql": {"id": "root"}}) as client: + cookies = {"ds_actor": client.actor_cookie({"id": "root"})} + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private", + cookies=cookies, + ) + assert response.status == 200 + assert response.json["private"] is True + # Anonymous users cannot execute SQL at all here + anon = client.get("/fixtures/-/query.json?sql=select+1+as+one") + assert anon.status == 403 + + +def test_query_extra_private_false_when_sql_is_public(): + with make_app_client() as client: + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ) + assert response.status == 200 + assert response.json["private"] is False + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From 8f888515b618bc0eb18e23c861b938b6bbbbf5d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:47:26 -0700 Subject: [PATCH 1842/1866] Fix _extra=query to report the params that were actually bound QueryExtra re-derived named parameters from the SQL with a regex, which missed parameters declared in a stored query's params list, reported magic _-prefixed parameters with raw querystring values that were never bound, and echoed the entire querystring when no SQL was present. QueryView now passes its named_parameter_values dict - the parameters it actually bound - through QueryExtraContext. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 2 +- datasette/views/table_extras.py | 11 +--------- tests/test_table_api.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index e6efddea..a719fa4f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -717,7 +717,7 @@ class QueryView(View): rows=rows, columns=columns, sql=sql, - params=params_for_query, + params=named_parameter_values, query_name=stored_query.name if stored_query else None, stored_query=stored_query, stored_query_write=stored_query_write, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 63c87a6f..21a908a0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,7 +8,6 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, - named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -592,17 +591,9 @@ class QueryExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - params = context.params - if context.scope == ExtraScope.QUERY and context.sql: - parameter_names = set(derive_named_parameters(context.sql)) - params = { - key: value - for key, value in dict(context.params).items() - if key in parameter_names - } return { "sql": context.sql, - "params": params, + "params": context.params, } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 4ab2f596..cfa3b512 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -131,6 +131,44 @@ def test_query_extra_private_for_arbitrary_sql(): assert anon.status == 403 +def test_query_extra_query_reports_bound_params(): + config = { + "databases": { + "fixtures": { + "queries": { + "declared_params": { + "sql": "select 1 as one", + "params": ["foo"], + }, + "magic_host": { + "sql": "select :_header_host as h", + }, + } + } + } + } + with make_app_client(config=config) as client: + # Declared parameters are reported even when the regex cannot find them + response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query") + assert response.status == 200 + assert response.json["query"]["params"] == {"foo": "bar"} + # Magic parameters are bound internally and should not be reported, + # especially not as a value taken from the querystring + response = client.get( + "/fixtures/magic_host.json?_extra=query&_header_host=spoofed" + ) + assert response.status == 200 + assert response.json["rows"] == [{"h": "localhost"}] + assert response.json["query"]["params"] == {} + + +def test_query_extra_query_does_not_echo_querystring_without_sql(): + with make_app_client() as client: + response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + assert response.status == 200 + assert response.json["query"]["params"] == {} + + def test_query_extra_private_false_when_sql_is_public(): with make_app_client() as client: response = client.get( From b635dc53f42e06908c7510d743e85100a6488f22 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:50:44 -0700 Subject: [PATCH 1843/1866] Make filters, actions and display_rows extras internal These three extras return values that exist for the HTML templates - a Filters instance, an async function and markupsafe/sqlite3.Row data - so requesting them on a .json page returned a 500 serialization error, while the generated documentation and ?_extra=extras both advertised them as API surface. They are now public=False: ignored like any unknown name on JSON requests, omitted from the docs and the extras list, and still resolved for the HTML view via the new include_internal flag on ExtraRegistry.resolve(). Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 17 ++++++++--------- datasette/views/table.py | 9 ++++++++- datasette/views/table_extras.py | 12 ++++++++++-- docs/json_api.rst | 9 --------- tests/test_table_api.py | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index f655e517..d5847937 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -89,7 +89,7 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) - async def resolve(self, requested, context, scope): + async def resolve(self, requested, context, scope, include_internal=False): registry = Registry() async def context_provider(): @@ -100,15 +100,14 @@ class ExtraRegistry: for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) - public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} - requested_public_names = [ - name - for name in requested - if name in public_names and name in registry._registry - ] - resolved = await registry.resolve_multi(requested_public_names) + allowed_names = { + cls.key() + for cls in self.classes_for_scope(scope, include_internal=include_internal) + } + requested_names = [name for name in requested if name in allowed_names] + resolved = await registry.resolve_multi(requested_names) return { - name: resolved[name] for name in requested_public_names if name in resolved + name: resolved[name] for name in requested_names if name in resolved } diff --git a/datasette/views/table.py b/datasette/views/table.py index 1b298c50..3cf8e6c6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1518,7 +1518,14 @@ async def table_view_data( "ok": True, "next": next_value and str(next_value) or None, } - data.update(await resolve_table_extras(extras, table_extra_context)) + data.update( + await resolve_table_extras( + extras, + table_extra_context, + # The HTML view needs extras that are not JSON serializable + include_internal=bool(extra_extras), + ) + ) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 21a908a0..c98ae22c 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -333,6 +333,8 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) + # Returns an async function for the HTML templates - not JSON serializable + public = False async def resolve(self, context): async def actions(): @@ -476,6 +478,8 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) + # Contains markupsafe/sqlite3.Row values - not JSON serializable + public = False async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["rows"] @@ -772,6 +776,8 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) + # Returns a Filters instance for the HTML templates - not JSON serializable + public = False async def resolve(self, context): return context.filters @@ -1034,8 +1040,10 @@ TABLE_EXTRA_CLASSES = [ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) -async def resolve_table_extras(extras, context): - return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) +async def resolve_table_extras(extras, context, include_internal=False): + return await table_extra_registry.resolve( + extras, context, ExtraScope.TABLE, include_internal=include_internal + ) async def resolve_row_extras(extras, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 379d26a0..6b595577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -425,9 +425,6 @@ The available table extras are listed below. } ] -``display_rows`` - Row data formatted for the HTML table display - ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook @@ -554,12 +551,6 @@ The available table extras are listed below. "9403e5" -``actions`` - Table or view actions made available by plugin hooks - -``filters`` - Filters object used by the HTML table interface - ``renderers`` Alternative output renderers available for this table diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cfa3b512..0cb67164 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,24 @@ async def test_query_extras_for_stored_query(ds_client): } +@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"]) +@pytest.mark.asyncio +async def test_html_only_extras_are_not_available_via_json(ds_client, extra): + # These extras exist for the HTML view; their values are not JSON + # serializable so they are internal, not part of the JSON API + response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") + assert response.status_code == 200 + assert extra not in response.json() + + +@pytest.mark.asyncio +async def test_html_only_extras_are_not_advertised(ds_client): + response = await ds_client.get("/fixtures/facetable.json?_extra=extras") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["extras"]} + assert {"filters", "actions", "display_rows"}.isdisjoint(names) + + def test_query_extra_private_for_arbitrary_sql(): with make_app_client(config={"allow_sql": {"id": "root"}}) as client: cookies = {"ds_actor": client.actor_cookie({"id": "root"})} From bbf0424c4519441715f73ee6468e0c53cc959861 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:51:25 -0700 Subject: [PATCH 1844/1866] Changelog for row/query extras and related fixes Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75e4f3e8..19089dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,13 @@ Changelog ------------------- - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) +- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. +- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. +- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. +- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. +- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. +- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. .. _v1_0_a32: From 6babd23cec9c41edd3d0ba2fab1c319905446b21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:53:00 -0700 Subject: [PATCH 1845/1866] QueryView: only resolve extras for renderer formats, single metadata path Extras were resolved before the format dispatch, so a .csv request carrying ?_extra= parameters paid for extras (including per-cell render_cell plugin calls) whose results were then discarded, and the HTML path duplicated the stored-query metadata derivation. Extras now resolve inside the renderer-dispatch branch only, and both consumers share a query_metadata() helper that no longer fetches database metadata just to throw it away for stored queries. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a719fa4f..ad3fb843 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -700,33 +700,12 @@ class QueryView(View): except DatasetteError: raise - extras = extra_names_from_request(request) - metadata = None - data = {"ok": True, "rows": rows, "columns": columns} - if extras: - metadata = await datasette.get_database_metadata(database) + async def query_metadata(): if stored_query: metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) - query_extra_context = QueryExtraContext( - datasette=datasette, - request=request, - db=db, - database_name=database, - private=private, - rows=rows, - columns=columns, - sql=sql, - params=named_parameter_values, - query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, - metadata=metadata, - extras=extras, - extra_registry=table_extra_registry, - ) - data.update(await resolve_query_extras(extras, query_extra_context)) + return metadata + return await datasette.get_database_metadata(database) # Handle formats from plugins if format_ == "csv": @@ -740,6 +719,28 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + data = {"ok": True, "rows": rows, "columns": columns} + extras = extra_names_from_request(request) + if extras: + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=named_parameter_values, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=await query_metadata(), + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -806,11 +807,7 @@ class QueryView(View): ) } ) - if metadata is None: - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) + metadata = await query_metadata() renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( From a1b6a6976d0ddafba6b927ce9fb83e62cb9091c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:55:28 -0700 Subject: [PATCH 1846/1866] Remove dead weight from the extras machinery - TableExtraContext.next_value, RowExtraContext.resolved and QueryExtraContext.stored_query/stored_query_write/error had no readers - drop the fields and the arguments that populated them - Extra.documentation() and the stable classvar were unused parallel descriptions of what the docs generator reads directly - ExtraRegistry.resolve no longer carries an always-true membership guard (resolve_multi returns every requested registered name) Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 19 +------------------ datasette/views/database.py | 3 --- datasette/views/row.py | 1 - datasette/views/table.py | 1 - datasette/views/table_extras.py | 5 ----- 5 files changed, 1 insertion(+), 28 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d5847937..4aa93057 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -50,7 +50,6 @@ class Extra(Provider): example: ClassVar[ExtraExample | None] = None examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True - stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None @@ -58,20 +57,6 @@ class Extra(Provider): def example_for_scope(cls, scope): return cls.examples.get(scope, cls.example) - @classmethod - def documentation(cls): - return { - "name": cls.key(), - "description": cls.description, - "scopes": [ - scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) - ], - "stable": cls.stable, - "expensive": cls.expensive, - "docs_note": cls.docs_note, - "example": cls.example, - } - class ExtraRegistry: def __init__(self, classes): @@ -106,9 +91,7 @@ class ExtraRegistry: } requested_names = [name for name in requested if name in allowed_names] resolved = await registry.resolve_multi(requested_names) - return { - name: resolved[name] for name in requested_names if name in resolved - } + return {name: resolved[name] for name in requested_names} def _camel_to_snake(name): diff --git a/datasette/views/database.py b/datasette/views/database.py index ad3fb843..46e26496 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -733,9 +733,6 @@ class QueryView(View): sql=sql, params=named_parameter_values, query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, metadata=await query_metadata(), extras=extras, extra_registry=table_extra_registry, diff --git a/datasette/views/row.py b/datasette/views/row.py index e15dfce9..c300758b 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -172,7 +172,6 @@ class RowView(DataView): row_extra_context = RowExtraContext( datasette=self.ds, request=request, - resolved=resolved, db=db, database_name=database, table_name=table, diff --git a/datasette/views/table.py b/datasette/views/table.py index 3cf8e6c6..65388c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1497,7 +1497,6 @@ async def table_view_data( nofacet=nofacet, nosuggest=nosuggest, next_arg=request.args.get("_next"), - next_value=next_value, next_url=next_url, sql=sql, sql_no_order_no_limit=sql_no_order_no_limit, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index c98ae22c..493135f3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -37,7 +37,6 @@ class TableExtraContext: nofacet: object nosuggest: object next_arg: object - next_value: object next_url: str | None sql: str sql_no_order_no_limit: str @@ -60,7 +59,6 @@ class TableExtraContext: class RowExtraContext: datasette: object request: object - resolved: object db: object database_name: str table_name: str @@ -90,9 +88,6 @@ class QueryExtraContext: sql: str | None params: dict query_name: str | None - stored_query: object - stored_query_write: bool - error: str | None metadata: dict extras: set extra_registry: ExtraRegistry From cfafa5b37f5350303600e912955c43e210a113b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:56:40 -0700 Subject: [PATCH 1847/1866] Use plain set literals for Extra scopes frozenset({...}) was immutability ceremony for class attributes that nothing mutates. scopes = {ExtraScope.TABLE} reads cleaner. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 2 +- datasette/views/table_extras.py | 74 ++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 4aa93057..fee92939 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -30,7 +30,7 @@ class ExtraExample: class Provider: name: ClassVar[str | None] = None - scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + scopes: ClassVar[set[ExtraScope]] = set() public: ClassVar[bool] = False @classmethod diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 493135f3..ce1d7bdf 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -100,7 +100,7 @@ class QueryExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.count_sql @@ -109,7 +109,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" example = ExtraExample("/fixtures/facetable.json?_extra=count") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context): @@ -141,7 +141,7 @@ class CountExtra(Extra): class FacetInstancesProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, count): facet_instances = [] @@ -182,7 +182,7 @@ class FacetResultsExtra(Extra): }, note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -217,7 +217,7 @@ class FacetsTimedOutExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): return facet_results["timed_out"] @@ -234,7 +234,7 @@ class SuggestedFacetsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -259,7 +259,7 @@ class HumanDescriptionEnExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): human_description_en = context.filters.human_description_en( @@ -279,7 +279,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.next_url @@ -296,7 +296,7 @@ class ColumnsExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.columns @@ -305,7 +305,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return list(context.table_columns) @@ -319,7 +319,7 @@ class PrimaryKeysExtra(Extra): "/fixtures/simple_primary_key/1.json?_extra=primary_keys" ) } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.pks @@ -327,7 +327,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns an async function for the HTML templates - not JSON serializable public = False @@ -358,7 +358,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.is_view @@ -375,7 +375,7 @@ class DebugExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): debug = { @@ -407,7 +407,7 @@ class RequestExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -420,7 +420,7 @@ class RequestExtra(Extra): class DisplayColumnsAndRowsProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): display_columns, display_rows = await context.display_columns_and_rows( @@ -464,7 +464,7 @@ class DisplayColumnsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["columns"] @@ -472,7 +472,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Contains markupsafe/sqlite3.Row values - not JSON serializable public = False @@ -524,7 +524,7 @@ class RenderCellExtra(Extra): ), ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): table_name = context.table_name @@ -587,7 +587,7 @@ class QueryExtra(Extra): ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), ], } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -599,7 +599,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -616,7 +616,7 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): description = "Column type UI metadata for this table" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context.is_view: @@ -676,7 +676,7 @@ class MetadataExtra(Extra): "/fixtures/neighborhood_search.json?text=town&_extra=metadata" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): if context.scope == ExtraScope.QUERY: @@ -713,7 +713,7 @@ class DatabaseExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.database_name @@ -725,7 +725,7 @@ class TableExtra(Extra): examples = { ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.table_name @@ -742,7 +742,7 @@ class DatabaseColorExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.db.color @@ -753,7 +753,7 @@ class FormHiddenArgsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): form_hidden_args = [] @@ -770,7 +770,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns a Filters instance for the HTML templates - not JSON serializable public = False @@ -781,7 +781,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return [ @@ -796,7 +796,7 @@ class SortedFacetResultsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): facet_configs = context.table_metadata.get("facets", []) @@ -832,7 +832,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_table_definition(context.table_name) @@ -841,7 +841,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_view_definition(context.table_name) @@ -850,7 +850,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" example = ExtraExample("/fixtures/facetable.json?_extra=renderers") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): renderers = {} @@ -896,7 +896,7 @@ class PrivateExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.private @@ -905,7 +905,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): expandables = [] @@ -921,7 +921,7 @@ class ForeignKeyTablesExtra(Extra): example = ExtraExample( "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) - scopes = frozenset({ExtraScope.ROW}) + scopes = {ExtraScope.ROW} async def resolve(self, context): return await context.foreign_key_tables( @@ -931,7 +931,7 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): all_extras = [ From 4edea3ad2637f4bf275f3a322e4c7747b964d907 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:04:12 -0700 Subject: [PATCH 1848/1866] Build extras registries once per scope instead of per request ExtraRegistry.resolve() previously constructed a fresh asyncinject Registry on every table, row and query request - instantiating all ~37 Extra classes and re-running inspect.signature reflection over each resolve method every time. The Extra classes are stateless, so the asyncinject Registry for each scope is now built lazily once and shared, along with the allowed-name sets. The per-request context reaches the shared registry through a contextvars.ContextVar provider rather than resolve_multi(results=...) seeding: asyncinject's parallel executor never schedules anything when the only initially-ready node is an unregistered pre-seeded value, so seeding would have stalled every resolution. asyncio tasks copy the caller's context, which keeps concurrent resolves isolated - covered by a new test. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 63 ++++++++++++++++++++++++++++++++---------- tests/test_extras.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/test_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index fee92939..2c3450b2 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,3 +1,4 @@ +import contextvars import re from dataclasses import dataclass from enum import Enum @@ -5,6 +6,11 @@ from typing import ClassVar from asyncinject import Registry +# Per-request context for Extra.resolve(), so the asyncinject registries can +# be shared across requests. asyncio tasks copy the caller's context, so +# concurrent resolve() calls each see their own value. +_resolve_context = contextvars.ContextVar("datasette_extras_context") + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -62,6 +68,13 @@ class ExtraRegistry: def __init__(self, classes): self.classes = list(classes) self.classes_by_name = {cls.key(): cls for cls in self.classes} + # Lazily-built shared state, keyed by scope. Safe to share across + # requests because Extra instances are stateless and asyncinject's + # Registry keeps per-call state local to each resolve_multi() call. + # If extras classes ever become registerable at runtime (e.g. via a + # plugin hook) these caches will need invalidating. + self._scope_registries = {} + self._allowed_names = {} def classes_for_scope(self, scope, include_internal=True): classes = [ @@ -74,23 +87,43 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def _registry_for_scope(self, scope): + registry = self._scope_registries.get(scope) + if registry is None: + registry = Registry() + + async def context_provider(): + return _resolve_context.get() + + registry.register(context_provider, name="context") + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + self._scope_registries[scope] = registry + return registry + + def _allowed_names_for_scope(self, scope, include_internal): + key = (scope, include_internal) + names = self._allowed_names.get(key) + if names is None: + names = { + cls.key() + for cls in self.classes_for_scope( + scope, include_internal=include_internal + ) + } + self._allowed_names[key] = names + return names + async def resolve(self, requested, context, scope, include_internal=False): - registry = Registry() - - async def context_provider(): - return context - - registry.register(context_provider, name="context") - - for cls in self.classes_for_scope(scope): - registry.register(cls().resolve, name=cls.key()) - - allowed_names = { - cls.key() - for cls in self.classes_for_scope(scope, include_internal=include_internal) - } + allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - resolved = await registry.resolve_multi(requested_names) + token = _resolve_context.set(context) + try: + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names + ) + finally: + _resolve_context.reset(token) return {name: resolved[name] for name in requested_names} diff --git a/tests/test_extras.py b/tests/test_extras.py new file mode 100644 index 00000000..ad8a9f00 --- /dev/null +++ b/tests/test_extras.py @@ -0,0 +1,65 @@ +import asyncio + +import pytest + +from datasette.extras import Extra, ExtraRegistry, ExtraScope + + +class SlowValueExtra(Extra): + description = "Returns context['value'], optionally slowly" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context["slow"]: + await asyncio.sleep(0.05) + return context["value"] + + +class DependentExtra(Extra): + description = "Depends on slow_value" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, slow_value): + return slow_value + 1 + + +def test_registry_is_built_once_per_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + first = registry._registry_for_scope(ExtraScope.TABLE) + second = registry._registry_for_scope(ExtraScope.TABLE) + assert first is second + + +@pytest.mark.asyncio +async def test_concurrent_resolves_do_not_share_state(): + # The asyncinject registry is shared across requests - resolved values + # must not leak between concurrent resolve() calls with different contexts + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + slow, fast = await asyncio.gather( + registry.resolve( + {"slow_value", "dependent"}, + {"value": 100, "slow": True}, + ExtraScope.TABLE, + ), + registry.resolve( + {"slow_value", "dependent"}, + {"value": 200, "slow": False}, + ExtraScope.TABLE, + ), + ) + assert slow == {"slow_value": 100, "dependent": 101} + assert fast == {"slow_value": 200, "dependent": 201} + + +@pytest.mark.asyncio +async def test_table_row_and_query_scopes_use_separate_registries(): + from datasette.views.table_extras import table_extra_registry + + registries = { + scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope + } + assert len(set(map(id, registries.values()))) == 3 + # Scope-specific extras only registered where they belong + assert "count" in registries[ExtraScope.TABLE]._registry + assert "count" not in registries[ExtraScope.QUERY]._registry + assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry From 96226621325c5aa19e6a700efcd1b441006958fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:15:18 -0700 Subject: [PATCH 1849/1866] Fix SQL injection via bracket escape bypass in escape_sqlite() (#2677) escape_sqlite() wrapped identifiers in [brackets] without escaping any ] characters inside the string. Since SQLite does not support escaping ] within bracket quoting, an identifier containing ] could break out and inject arbitrary SQL. Fall back to double-quote quoting (doubling any embedded ") when the identifier contains ]. Co-Authored-By: Claude Fable 5 --- datasette/utils/__init__.py | 4 ++++ tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2dff9667..55e539b9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -410,6 +410,10 @@ def escape_css_string(s): def escape_sqlite(s): if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s + elif "]" in s: + # SQLite does not support escaping ] inside [bracket] quoting, so fall + # back to double-quote quoting (doubling any embedded ") - #2677 + return '"{}"'.format(s.replace('"', '""')) else: return f"[{s}]" diff --git a/tests/test_utils.py b/tests/test_utils.py index 64607244..74f1963f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -216,6 +216,38 @@ def test_detect_fts(open_quote, close_quote): conn.close() +@pytest.mark.parametrize( + "identifier,expected", + ( + ("plain", "plain"), + ("select", "[select]"), + ("has space", "[has space]"), + ("has'quote", "[has'quote]"), + # Identifiers containing ] must fall back to double-quote quoting + # (SQLite does not support escaping ] inside [brackets]) - #2677 + ("has]bracket", '"has]bracket"'), + ('has"dquote]', '"has""dquote]"'), + ), +) +def test_escape_sqlite(identifier, expected): + assert utils.escape_sqlite(identifier) == expected + + +def test_escape_sqlite_prevents_injection(): + # https://github.com/simonw/datasette/issues/2677 + conn = utils.sqlite3.connect(":memory:") + conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") + conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") + malicious = "users] UNION SELECT password FROM users--" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + results = conn.execute(sql).fetchall() + conn.close() + # The injected UNION must not execute - only the empty malicious table + # is queried, so we get a single count row and no leaked password + assert results == [(0,)] + + @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): sql = """ From 1c514d69f6cc09c820c119e7bbf4dc75235e90cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:17:16 -0700 Subject: [PATCH 1850/1866] Prevent open redirect via backslash in path (#2680) asgi_send_redirect() only collapsed leading forward slashes, so a path like /\example.com/ produced a Location of /\example.com. Browsers normalise backslashes to forward slashes, turning that into the protocol-relative //example.com and redirecting off-site. Collapse any run of leading slashes and backslashes to a single slash. Co-Authored-By: Claude Fable 5 --- datasette/utils/asgi.py | 8 +++++--- tests/test_custom_pages.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 35f243b6..55eba1bb 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -330,9 +330,11 @@ async def asgi_send_html(send, html, status=200, headers=None): async def asgi_send_redirect(send, location, status=302): - # Prevent open redirect vulnerability: strip multiple leading slashes - # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/) - location = re.sub(r"^/+", "/", location) + # Prevent open redirect vulnerability: collapse leading slashes and + # backslashes down to a single slash. //example.com is a protocol-relative + # URL, and browsers normalise backslashes to slashes so /\example.com would + # be treated as //example.com - https://github.com/simonw/datasette/issues/2680 + location = re.sub(r"^[/\\]+", "/", location) await asgi_send( send, "", diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 39a4c06b..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -104,3 +104,24 @@ def test_custom_route_pattern_with_slash_slash_302(custom_pages_client): response = custom_pages_client.get("//example.com/") assert response.status == 302 assert response.headers["location"] == "/example.com" + + +@pytest.mark.parametrize( + "path", + ( + "/\\example.com/", + "/\\\\example.com/", + "/\\/example.com/", + ), +) +def test_redirect_does_not_allow_backslash_open_redirect(custom_pages_client, path): + # https://github.com/simonw/datasette/issues/2680 + # Browsers normalise backslashes to forward slashes, so a Location of + # /\example.com would be treated as the protocol-relative //example.com + response = custom_pages_client.get(path) + assert response.status == 302 + location = response.headers["location"] + assert location == "/example.com" + # Must not start with anything a browser reads as protocol-relative + assert not location.startswith("//") + assert not location.startswith("/\\") From c31bb55011567d13f39d8096da4aef5b5a8a720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:24:21 -0700 Subject: [PATCH 1851/1866] Add regression test for --default-deny index 500 (#2644) datasette --default-deny --root with no config file previously 500'd on the instance and database index pages: rendering them computes is_private (include_is_private=True), which references the anon_rules CTE, but that CTE was only defined when anonymous permission rules existed. This was fixed by the empty-anon_rules fallback added in 4b5fac9c; this commit adds a regression test that fails without that fallback (SQLite "no such table: anon_rules" -> 500). Co-Authored-By: Claude Fable 5 --- tests/test_default_deny.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index 81e95b84..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -127,3 +127,23 @@ async def test_default_deny_basic_permissions(): # Authenticated user without explicit permission should also be denied assert await ds.allowed(action="view-instance", actor={"id": "user"}) is False + + +@pytest.mark.asyncio +async def test_default_deny_root_no_config_index_does_not_500(): + # https://github.com/simonw/datasette/issues/2644 + # --default-deny --root with no config file must not 500 on the index + # pages. Rendering those pages computes is_private (include_is_private), + # which references the anon_rules CTE - that CTE must still be defined + # even when there are no anonymous permission rules at all. + ds = Datasette(default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + db = ds.add_memory_database("test_db_2644") + await db.execute_write("create table test_table (id integer primary key)") + await ds._refresh_schemas() + + cookie = ds.sign({"a": {"id": "root"}}, "actor") + for path in ("/", "/test_db_2644", "/test_db_2644/test_table"): + response = await ds.client.get(path, cookies={"ds_actor": cookie}) + assert response.status_code == 200, f"{path} returned {response.status_code}" From d5141a5778ac5ce6a6f4cfda990b2ab556b7f9f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:27:13 -0700 Subject: [PATCH 1852/1866] Fix /-/check 500 for query actions (#2756) _check_permission_for_actor() constructed child resources with resource_class(database=parent, table=child), but QueryResource takes a "query" argument, not "table", so /-/check?action=delete-query (and view-query / update-query) raised TypeError. Construct the resource positionally so it works for any child resource class. Co-Authored-By: Claude Fable 5 --- datasette/views/special.py | 8 +++++--- tests/test_permissions.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 75c54c3c..aa063ad6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -497,11 +497,13 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): if action_obj.resource_class is None: resource_obj = None elif action_obj.takes_parent and action_obj.takes_child: - # Child-level resource (e.g., TableResource, QueryResource) - resource_obj = action_obj.resource_class(database=parent, table=child) + # Child-level resource (e.g., TableResource, QueryResource). The child + # argument is named differently per resource class (table, query, ...), + # so pass positionally - https://github.com/simonw/datasette/issues/2756 + resource_obj = action_obj.resource_class(parent, child) elif action_obj.takes_parent: # Parent-level resource (e.g., DatabaseResource) - resource_obj = action_obj.resource_class(database=parent) + resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ return {"error": f"Invalid action configuration: {action}"}, 500 diff --git a/tests/test_permissions.py b/tests/test_permissions.py index e5e75432..8323fe92 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1733,6 +1733,29 @@ async def test_permission_check_view_requires_debug_permission(): assert data["allowed"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) +async def test_permission_check_view_query_actions(action): + # https://github.com/simonw/datasette/issues/2756 + # QueryResource takes a "query" argument, not "table", so /-/check must + # not assume every child resource class accepts table= + ds = Datasette() + ds.root_enabled = True + root_token = await ds.create_token("root", handler="signed") + response = await ds.client.get( + f"/-/check.json?action={action}&parent=mydb&child=myquery", + headers={"Authorization": f"Bearer {root_token}"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["action"] == action + assert data["resource"] == { + "parent": "mydb", + "child": "myquery", + "path": "/mydb/myquery", + } + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 154ea483eaba7a636289a6972baeced7163acd60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:30:12 -0700 Subject: [PATCH 1853/1866] Pass columns and rows to can_render for canned queries (#2711) The HTML branch of QueryView built an empty data dict before looping over register_output_renderer can_render callbacks, so renderers that depend on the result columns or rows (e.g. datasette-atom, datasette-ics) never appeared as export options for canned queries. Populate data with the executed query's rows, columns, SQL and query name. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 10 +++++++++- tests/test_plugins.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..66887f9b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -762,7 +762,15 @@ class QueryView(View): ) ), ) - data = {} + data = { + "ok": query_error is None, + "rows": rows, + "columns": columns, + "query": {"sql": sql, "params": params}, + "query_name": stored_query.name if stored_query else None, + "database": database, + "table": None, + } headers.update( { "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 32276437..cf753c9e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -626,6 +626,31 @@ async def test_hook_register_output_renderer_can_render(ds_client): }.items() <= ds_client.ds._can_render_saw.items() +@pytest.mark.asyncio +async def test_hook_register_output_renderer_can_render_canned_query(ds_client): + # https://github.com/simonw/datasette/issues/2711 + # can_render for a canned query must be passed the query's columns, rows + # and SQL - previously it received an empty data dict, so renderers that + # depend on the columns (datasette-atom, datasette-ics) never showed up. + response = await ds_client.get("/fixtures/pragma_cache_size") + assert response.status_code == 200 + saw = ds_client.ds._can_render_saw + assert saw["columns"] == ["cache_size"] + assert len(saw["rows"]) == 1 + assert saw["sql"] == "PRAGMA cache_size;" + assert saw["query_name"] == "pragma_cache_size" + # The renderer's export link should therefore be offered + links = ( + Soup(response.text, "html.parser") + .find("p", {"class": "export-links"}) + .find_all("a") + ) + actual = [link["href"] for link in links] + assert any( + href.startswith("/fixtures/pragma_cache_size.testall") for href in actual + ) + + @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" From 92848c06b8cf5b8bb6b93088bb547810f9fce8b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:43:32 -0700 Subject: [PATCH 1854/1866] Stop facet counts from wrapping (#2754) ul.tight-bullets li uses word-break: break-all so long facet labels can wrap, but that also let the count number break across lines. Wrap each count in a span.facet-count with white-space: nowrap so the label can still wrap while the count stays on one line. Co-Authored-By: Claude Fable 5 --- datasette/static/app.css | 5 +++++ datasette/templates/_facet_results.html | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..6d675d9f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -706,6 +706,11 @@ button.core[type=button] { color: #666; padding-right: 0.25em; } +/* The label may wrap (word-break: break-all on the li) but the count should + stay on one line - https://github.com/simonw/datasette/issues/2754 */ +.facet-count { + white-space: nowrap; +} .facet-info li, .facet-info ul { margin: 0; diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@
      {% for facet_value in facet_info.results %} {% if not facet_value.selected %} -
    • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
      • +
    • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
      • {% else %} -
    • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
      • +
    • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
      • {% endif %} {% endfor %} {% if facet_info.truncated %} From 9adb5416743a7312758e986c254baa1758228ad0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:42:08 -0700 Subject: [PATCH 1855/1866] Use asyncinject 0.7 results= seeding for per-request extras context asyncinject 0.7 fixed the parallel executor stalling when every initially-ready node is a seeded value, and made seeded values take precedence over registered functions. That lets the shared per-scope registries receive the per-request context directly via resolve_multi(results={'context': ...}) instead of the contextvars.ContextVar workaround. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 21 +++------------------ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 2c3450b2..5cab52a4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,3 @@ -import contextvars import re from dataclasses import dataclass from enum import Enum @@ -6,11 +5,6 @@ from typing import ClassVar from asyncinject import Registry -# Per-request context for Extra.resolve(), so the asyncinject registries can -# be shared across requests. asyncio tasks copy the caller's context, so -# concurrent resolve() calls each see their own value. -_resolve_context = contextvars.ContextVar("datasette_extras_context") - def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -91,11 +85,6 @@ class ExtraRegistry: registry = self._scope_registries.get(scope) if registry is None: registry = Registry() - - async def context_provider(): - return _resolve_context.get() - - registry.register(context_provider, name="context") for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) self._scope_registries[scope] = registry @@ -117,13 +106,9 @@ class ExtraRegistry: async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - token = _resolve_context.set(context) - try: - resolved = await self._registry_for_scope(scope).resolve_multi( - requested_names - ) - finally: - _resolve_context.reset(token) + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names, results={"context": context} + ) return {name: resolved[name] for name in requested_names} diff --git a/pyproject.toml b/pyproject.toml index 38085476..0d136d60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6.1", + "asyncinject>=0.7", "setuptools", "pip", ] From 648a34ce8196ecf02504c0daed594bd1cd540210 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:13:07 -0700 Subject: [PATCH 1856/1866] Fix for test I broke in 92848c06 refs #2754 --- tests/test_table_html.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2e671d55..63e233fa 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -349,7 +349,11 @@ async def test_facet_display(ds_client): { "name": a.text, "qs": a["href"].split("?")[-1], - "count": int(str(a.parent).split("")[1].split("<")[0]), + "count": int( + a.parent.find( + "span", {"class": "facet-count"} + ).text.replace(",", "") + ), } for a in div.find("ul").find_all("a") ], @@ -695,7 +699,7 @@ async def test_table_html_foreign_key_facets(ds_client): assert response.status_code == 200 assert ( '
    • - 1
      • ' + ' data-facet-value="3">- 1' ) in response.text From 26f3b20e58bd2ad582d5fae326acf22b42627eb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:29:27 -0700 Subject: [PATCH 1857/1866] Fix to our pytest plugin to better support pytest-cov Refs https://github.com/simonw/datasette/pulls#issuecomment-4681621052 --- datasette/_pytest_plugin.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): From 993169ae496aa0fa30271b6cb4dfc50202f6e7c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 08:24:37 -0700 Subject: [PATCH 1858/1866] Release 1.0a33 Refs #2735, #2677, #2680, #2711, #2756, #2761, #2768, #2754 --- datasette/version.py | 2 +- docs/changelog.rst | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 1e8c61d5..9536d459 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a32" +__version__ = "1.0a33" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19089dd1..48bef0bf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,15 +9,43 @@ Changelog 1.0a33 (unreleased) ------------------- -- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) -- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. -- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. -- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. -- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. -- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. + +Editing and deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + +``?_extra=`` support for row and query pages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. + +New generated reference documentation describes every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra` for the full list. + +You can explore the new extras using this `Datasette extras API explorer tool `__. + +Other improvements and fixes to the extras mechanism: + - Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. - The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. +Security fixes +~~~~~~~~~~~~~~ + +- Fixed an identifier-quoting bug in ``datasette.utils.escape_sqlite()``. Datasette uses this helper when constructing SQL around table and column names; identifiers containing ``]`` could break out of SQLite bracket quoting and alter the generated SQL, for example by adding a ``UNION SELECT``. Identifiers containing ``]`` are now quoted using double quotes instead. (:issue:`2677`) +- Fixed an open redirect vulnerability. Requesting a path such as ``/\example.com/`` produced a redirect with a ``Location: /\example.com`` header - browsers normalize backslashes to forward slashes, turning that into the protocol-relative URL ``//example.com`` and redirecting the user off-site. Any run of leading slashes and backslashes in a redirect path is now collapsed to a single slash. (:issue:`2680`) + +Bug fixes +~~~~~~~~~ + +- ``can_render()`` callbacks registered by the :ref:`register_output_renderer() ` plugin hook now receive the result ``rows`` and ``columns`` for stored queries. Previously renderers that inspect the available columns - such as `datasette-atom `__ and `datasette-ics `__ - never appeared as export options on stored query pages. (:issue:`2711`) +- Fixed a 500 error from the :ref:`/-/check ` permission debugging endpoint when checking query actions such as ``view-query``, ``update-query`` and ``delete-query``. (:issue:`2756`) +- Write queries that use a named parameter called ``:sql`` no longer fail with an error. (:issue:`2761`) +- :ref:`db.execute_isolated_fn() ` now works against immutable databases, using a read-only connection that bypasses the write thread. It previously always attempted to open a writable connection, which would fail - breaking features built on top of it, such as the SQL analysis step used when storing a query. An exception raised while opening the connection for an isolated function no longer crashes the write thread. (:issue:`2768`) +- Facet counts are now displayed on the same line as the facet value instead of wrapping onto a second line. (:issue:`2754`) +- Datasette's pytest plugin no longer imports the rest of Datasette at pytest startup time. This means plugin test suites using ``pytest-cov`` now correctly record coverage of code that runs when ``datasette`` modules are first imported. + .. _v1_0_a32: 1.0a32 (2026-05-31) From 1d4212122e5597f2e13625193fb7d45b25928447 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 10:36:16 -0700 Subject: [PATCH 1859/1866] Add release date for 1.0a33 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48bef0bf..c0bd7e6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a33: -1.0a33 (unreleased) +1.0a33 (2026-06-11) ------------------- Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. From fa86ac7b11c44ef80146db6eed25d88c954ee37a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 19:41:24 -0700 Subject: [PATCH 1860/1866] Clearer examples and descriptions for JSON API extras (#2773) Review of the generated ?_extra= documentation found several extras with no example output or with examples that needed explanation: - extras: now shows an abbreviated example of the toggle list and has a clearer description (which also improves the live API output) - set_column_type_ui: example of the shape seen with set-column-type permission, plus a note that it is null otherwise - column_types: live example generated from a table with an assigned column type instead of an empty {} - metadata: live table example now demonstrates a table description and column descriptions; row and query examples gained explanatory notes - expandable_columns, foreign_key_tables, facets_timed_out, next_url, renderers: notes explaining the shape of their output Also added docs_note cross-references to the relevant documentation: facets, pagination, render_cell and register_output_renderer plugin hooks, column type configuration and API, metadata, custom templates, permissions and foreign key label expansion. foreign_key_tables is now flagged as potentially executing additional queries. https://claude.ai/code/session_01EfjBe6E817m9XNFW7EX3Vm Co-authored-by: Claude --- datasette/views/table_extras.py | 182 +++++++++++++++++++++++++-- docs/json_api.rst | 215 ++++++++++++++++++++++++++------ docs/json_api_doc.py | 19 ++- 3 files changed, 367 insertions(+), 49 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ce1d7bdf..948f3daa 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -184,6 +184,7 @@ class FacetResultsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = "See :ref:`facets` for details of how facets work." async def resolve(self, context, facet_instances): facet_results = {} @@ -215,7 +216,12 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" example = ExtraExample( - "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out", + note=( + "A list of the names of any facets that exceeded the " + ":ref:`setting_facet_time_limit_ms` time limit - an empty list " + "if every facet calculation completed." + ), ) scopes = {ExtraScope.TABLE} @@ -236,6 +242,9 @@ class SuggestedFacetsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = ( + "Suggestions are controlled by the :ref:`setting_suggest_facets` setting." + ) async def resolve(self, context, facet_instances): suggested_facets = [] @@ -278,7 +287,13 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" - example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") + example = ExtraExample( + "/fixtures/facetable.json?_size=1&_extra=next_url", + note=( + "``null`` if there are no more pages of results. " + "See :ref:`json_api_pagination`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -366,6 +381,10 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + docs_note = ( + "The contents of this block are not a stable part of the Datasette " + "API and may change without warning." + ) example = ExtraExample("/fixtures/facetable.json?_extra=debug") examples = { ExtraScope.ROW: ExtraExample( @@ -482,6 +501,10 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + docs_note = ( + "See the :ref:`render_cell() plugin hook ` " + "documentation." + ) example = ExtraExample( value={ "rows": [ @@ -598,7 +621,28 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" - example = ExtraExample(value={}) + docs_note = ( + "An empty object if no column types have been assigned. Column types " + "can be assigned in :ref:`configuration " + "` or using the :ref:`set column " + "type API `." + ) + example = ExtraExample( + "/fixtures/facetable.json?_size=0&_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column has " + "been assigned the ``json`` column type." + ), + ) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/facetable/1.json?_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column " + "has been assigned the ``json`` column type." + ), + ) + } scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): @@ -615,7 +659,40 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): - description = "Column type UI metadata for this table" + description = "Information needed to build an interface for assigning column types" + docs_note = ( + "``null`` unless the current actor is allowed to use the :ref:`set " + "column type API ` for this table." + ) + example = ExtraExample( + value={ + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": None, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + "tags": { + "current": {"type": "json", "config": None}, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + }, + }, + note=( + "Shape abbreviated to two columns, as seen by an actor with " + "``set-column-type`` permission. ``current`` is the column type " + "currently assigned to each column and ``options`` lists the " + "types that could be assigned to it." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -667,13 +744,33 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table, database or stored query" - example = ExtraExample("/fixtures/facetable.json?_extra=metadata") + docs_note = "See :ref:`metadata` for how to attach metadata to tables." + example = ExtraExample( + "/fixtures/facetable.json?_extra=metadata", + note=( + "This example is from an instance where the ``facetable`` table " + "has a metadata ``description`` and a :ref:`column description " + "` for its ``state`` column. The " + "``columns`` object is empty for tables with no column " + "descriptions." + ), + ) examples = { ExtraScope.ROW: ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=metadata" + "/fixtures/simple_primary_key/1.json?_extra=metadata", + note=( + "This table has no metadata, so only an empty ``columns`` " + "object is returned." + ), ), ExtraScope.QUERY: ExtraExample( - "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + "/fixtures/neighborhood_search.json?text=town&_extra=metadata", + note=( + "For stored queries this returns the full configuration of " + "the query, including the :ref:`stored query options " + "`. For ``?sql=`` queries it returns an " + "empty object." + ), ), } scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} @@ -733,6 +830,10 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + docs_note = ( + "A six character hex color, without the leading ``#``, derived from " + "a hash of the database name and used in the Datasette interface." + ) example = ExtraExample("/fixtures/facetable.json?_extra=database_color") examples = { ExtraScope.ROW: ExtraExample( @@ -780,6 +881,11 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + docs_note = ( + "The first template in this list that exists will be used to render " + "the table on the HTML version of this page. See " + ":ref:`customization_custom_templates`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = {ExtraScope.TABLE} @@ -793,6 +899,12 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + docs_note = ( + "The same data as ``facet_results``, as a list in the order used by " + "the HTML interface: facets from :ref:`facet configuration " + "` first, then other facets ordered by their number " + "of results." + ) example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) @@ -849,7 +961,15 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" - example = ExtraExample("/fixtures/facetable.json?_extra=renderers") + example = ExtraExample( + "/fixtures/facetable.json?_extra=renderers", + note=( + "Each key is the name of an output format, each value the URL " + "for this data in that format. Plugins can add additional " + "formats using the :ref:`register_output_renderer() plugin hook " + "`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): @@ -887,6 +1007,10 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this resource is private to the current actor" + docs_note = ( + "``true`` if the current actor can see this resource but an " + "anonymous user could not. See :ref:`authentication_permissions`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=private") examples = { ExtraScope.ROW: ExtraExample( @@ -904,7 +1028,15 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" - example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") + docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." + example = ExtraExample( + "/fixtures/facetable.json?_extra=expandable_columns", + note=( + "Each item is a ``[foreign_key, label_column]`` pair: the " + "foreign key relationship, then the column in the other table " + "that would be used as the label for each expanded value." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -919,9 +1051,14 @@ class ExpandableColumnsExtra(Extra): class ForeignKeyTablesExtra(Extra): description = "Tables that link to this row using foreign keys" example = ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables", + note=( + "``count`` is the number of rows in the other table that " + "reference this row, and ``link`` is a URL to browse those rows." + ), ) scopes = {ExtraScope.ROW} + expensive = True async def resolve(self, context): return await context.foreign_key_tables( @@ -930,7 +1067,30 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): - description = "Available ?_extra= blocks" + description = "List of ?_extra= blocks that can be used on this page" + example = ExtraExample( + value=[ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": False, + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": True, + }, + ], + note=( + "Shape abbreviated from /fixtures/facetable.json?_extra=extras - " + "the full response lists every extra described on this page. " + "``toggle_url`` is the current URL with that extra added or " + "removed, and ``selected`` is ``true`` for extras included in " + "the current request." + ), + ) scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 6b595577..fbc3cf60 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -276,7 +276,7 @@ The available table extras are listed below. "select count(*) from facetable " ``facet_results`` - Results of facets calculated against this data (May execute additional queries.) + Results of facets calculated against this data (May execute additional queries. See :ref:`facets` for details of how facets work.) Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. @@ -309,12 +309,14 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + A list of the names of any facets that exceeded the :ref:`setting_facet_time_limit_ms` time limit - an empty list if every facet calculation completed. + .. code-block:: json [] ``suggested_facets`` - Suggestions for facets that might return interesting results (May execute additional queries.) + Suggestions for facets that might return interesting results (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. @@ -341,6 +343,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. + .. code-block:: json "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" @@ -426,7 +430,7 @@ The available table extras are listed below. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -452,7 +456,7 @@ The available table extras are listed below. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/facetable.json?_extra=debug`` @@ -501,28 +505,108 @@ The available table extras are listed below. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) - .. code-block:: json + ``GET /fixtures/facetable.json?_size=0&_extra=column_types`` - {} - -``set_column_type_ui`` - Column type UI metadata for this table - -``metadata`` - Metadata about the table, database or stored query - - ``GET /fixtures/facetable.json?_extra=metadata`` + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json { - "columns": {} + "tags": { + "type": "json", + "config": null + } + } + +``set_column_type_ui`` + Information needed to build an interface for assigning column types (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) + + Shape abbreviated to two columns, as seen by an actor with ``set-column-type`` permission. ``current`` is the column type currently assigned to each column and ``options`` lists the types that could be assigned to it. + + .. code-block:: json + + { + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": null, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + }, + "tags": { + "current": { + "type": "json", + "config": null + }, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + } + } + } + +``metadata`` + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + + ``GET /fixtures/facetable.json?_extra=metadata`` + + This example is from an instance where the ``facetable`` table has a metadata ``description`` and a :ref:`column description ` for its ``state`` column. The ``columns`` object is empty for tables with no column descriptions. + + .. code-block:: json + + { + "description": "A demo table of places, used to demonstrate facets", + "columns": { + "state": "Two letter US state code" + } } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -543,7 +627,7 @@ The available table extras are listed below. "facetable" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/facetable.json?_extra=database_color`` @@ -556,6 +640,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_extra=renderers`` + Each key is the name of an output format, each value the URL for this data in that format. Plugins can add additional formats using the :ref:`register_output_renderer() plugin hook `. + .. code-block:: json { @@ -563,7 +649,7 @@ The available table extras are listed below. } ``custom_table_templates`` - Custom template names considered for this table + Custom template names considered for this table (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) ``GET /fixtures/facetable.json?_extra=custom_table_templates`` @@ -576,7 +662,7 @@ The available table extras are listed below. ] ``sorted_facet_results`` - Facet results sorted for display + Facet results sorted for display (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` @@ -643,7 +729,7 @@ The available table extras are listed below. true ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/facetable.json?_extra=private`` @@ -652,10 +738,12 @@ The available table extras are listed below. false ``expandable_columns`` - Foreign key columns that can be expanded with labels + Foreign key columns that can be expanded with labels (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` + Each item is a ``[foreign_key, label_column]`` pair: the foreign key relationship, then the column in the other table that would be used as the label for each expanded value. + .. code-block:: json [ @@ -720,7 +808,7 @@ The following extras are available for row JSON responses. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -741,7 +829,7 @@ The following extras are available for row JSON responses. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` @@ -803,17 +891,28 @@ The following extras are available for row JSON responses. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + + ``GET /fixtures/facetable/1.json?_extra=column_types`` + + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json - {} + { + "tags": { + "type": "json", + "config": null + } + } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + This table has no metadata, so only an empty ``columns`` object is returned. + .. code-block:: json { @@ -821,7 +920,26 @@ The following extras are available for row JSON responses. } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -842,7 +960,7 @@ The following extras are available for row JSON responses. "simple_primary_key" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` @@ -851,7 +969,7 @@ The following extras are available for row JSON responses. "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/simple_primary_key/1.json?_extra=private`` @@ -860,10 +978,12 @@ The following extras are available for row JSON responses. false ``foreign_key_tables`` - Tables that link to this row using foreign keys + Tables that link to this row using foreign keys (May execute additional queries.) ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + ``count`` is the number of rows in the other table that reference this row, and ``link`` is a URL to browse those rows. + .. code-block:: json [ @@ -921,7 +1041,7 @@ The following extras are available for arbitrary SQL query responses and stored, ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -941,7 +1061,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` @@ -1000,10 +1120,12 @@ The following extras are available for arbitrary SQL query responses and stored, } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + For stored queries this returns the full configuration of the query, including the :ref:`stored query options `. For ``?sql=`` queries it returns an empty object. + .. code-block:: json { @@ -1029,7 +1151,26 @@ The following extras are available for arbitrary SQL query responses and stored, } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -1041,7 +1182,7 @@ The following extras are available for arbitrary SQL query responses and stored, "fixtures" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` @@ -1050,7 +1191,7 @@ The following extras are available for arbitrary SQL query responses and stored, "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 44ef4a42..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -93,9 +93,26 @@ async def _fetch_live_examples(scoped_classes): datasette = Datasette( [str(db_path)], settings={"num_sql_threads": 1}, + metadata={ + "databases": { + "fixtures": { + "tables": { + "facetable": { + "description": "A demo table of places, used to demonstrate facets", + "columns": {"state": "Two letter US state code"}, + } + } + } + } + }, config={ "databases": { "fixtures": { + "tables": { + "facetable": { + "column_types": {"tags": "json"}, + } + }, "queries": { "neighborhood_search": { "sql": textwrap.dedent(""" @@ -108,7 +125,7 @@ async def _fetch_live_examples(scoped_classes): """), "title": "Search neighborhoods", } - } + }, } } }, From 88878b418473dcb399de376658d2bd7423b66c97 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 12:51:40 -0700 Subject: [PATCH 1861/1866] datasette.allowed_many() method --- datasette/app.py | 101 +++++++--- datasette/utils/actions_sql.py | 221 ++++++++++++++------- docs/internals.rst | 33 ++++ docs/plugin_hooks.rst | 6 + tests/conftest.py | 11 +- tests/test_allowed_many.py | 341 +++++++++++++++++++++++++++++++++ 6 files changed, 613 insertions(+), 100 deletions(-) create mode 100644 tests/test_allowed_many.py diff --git a/datasette/app.py b/datasette/app.py index 81d23acb..a6696ad9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -1817,46 +1817,97 @@ class Datasette: # For global actions, resource can be omitted: can_debug = await datasette.allowed(action="permissions-debug", actor=actor) """ - from datasette.utils.actions_sql import check_permission_for_resource + results = await self.allowed_many( + actions=[action], resource=resource, actor=actor + ) + return results[action] - # For global actions, resource remains None + async def allowed_many( + self, + *, + actions: Sequence[str], + resource: "Resource" = None, + actor: dict | None = None, + ) -> dict[str, bool]: + """ + Check several actions against one resource for one actor. - # Check if this action has also_requires - if so, check that action first - action_obj = self.actions.get(action) - if action_obj and action_obj.also_requires: - # Must have the required action first - if not await self.allowed( - action=action_obj.also_requires, - resource=resource, + Resolves every action (plus any also_requires dependencies) with a + single internal database query, instead of one or two queries per + action. + + Example: + from datasette.resources import TableResource + results = await datasette.allowed_many( + actions=["edit-schema", "drop-table", "insert-row"], + resource=TableResource(database="data", table="exercise"), actor=actor, - ): - return False + ) + # {"edit-schema": True, "drop-table": True, "insert-row": False} + """ + from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None child = resource.child if resource else None - result = await check_permission_for_resource( + # Expand also_requires dependencies (transitively) so that each + # dependency is resolved within the same batch + expanded = [] + + def add_action(name): + if name in expanded: + return + action_obj = self.actions.get(name) + if action_obj is None: + raise ValueError(f"Unknown action: {name}") + expanded.append(name) + if action_obj.also_requires: + add_action(action_obj.also_requires) + + requested = list(dict.fromkeys(actions)) + for name in requested: + add_action(name) + + raw = await check_permissions_for_actions( datasette=self, actor=actor, - action=action, + actions=expanded, parent=parent, child=child, ) + final = {} - # Log the permission check for debugging - self._permission_checks.append( - PermissionCheck( - when=datetime.datetime.now(datetime.timezone.utc).isoformat(), - actor=actor, - action=action, - parent=parent, - child=child, - result=result, + def resolve(name): + # final verdict = own rules AND verdict of also_requires chain + if name in final: + return final[name] + result = raw[name] + action_obj = self.actions.get(name) + if result and action_obj.also_requires: + result = resolve(action_obj.also_requires) + final[name] = result + return result + + for name in expanded: + resolve(name) + + # Log every check for the debug page, dependencies before the + # actions that required them + when = datetime.datetime.now(datetime.timezone.utc).isoformat() + for name in reversed(expanded): + self._permission_checks.append( + PermissionCheck( + when=when, + actor=actor, + action=name, + parent=parent, + child=child, + result=final[name], + ) ) - ) - return result + return {name: final[name] for name in requested} async def ensure_permission( self, diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index 891ee913..a422c1ed 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -21,6 +21,8 @@ The core pattern is: - Across levels, child beats parent beats global """ +import asyncio +import re from typing import TYPE_CHECKING from datasette.utils.permissions import gather_permission_sql_from_hooks @@ -495,6 +497,146 @@ async def build_permission_rules_sql( return rules_union, all_params, restriction_sqls +async def check_permissions_for_actions( + *, + datasette: "Datasette", + actor: dict | None, + actions: list[str], + parent: str | None, + child: str | None, +) -> dict[str, bool]: + """ + Check several actions for one actor and resource in a single query. + + Args: + datasette: The Datasette instance + actor: The actor dict (or None) + actions: List of action names to check + parent: The parent resource identifier (e.g., database name, or None) + child: The child resource identifier (e.g., table name, or None) + + Returns: + Dict mapping each action name to True (allowed) or False (denied) + + Each action contributes its own tagged block of permission rules + (gathered from the permission_resources_sql hook, with parameters + namespaced per action to avoid collisions) plus an optional + restriction allowlist CTE. One internal database query resolves + the winning rule per action using the same specificity-then-deny + ordering as the rest of the permission system. + + Note: this resolves each action independently - also_requires + dependencies are handled by the caller (Datasette.allowed_many). + """ + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + for action in actions: + if not datasette.actions.get(action): + raise ValueError(f"Unknown action: {action}") + + # Dedupe while preserving order + unique_actions = list(dict.fromkeys(actions)) + if not unique_actions: + return {} + + # Gather hook results for each action concurrently - hooks within a + # single action still run sequentially, preserving existing semantics + gathered = await asyncio.gather( + *( + gather_permission_sql_from_hooks( + datasette=datasette, actor=actor, action=action + ) + for action in unique_actions + ) + ) + + if any(result is SKIP_PERMISSION_CHECKS for result in gathered): + return {action: True for action in unique_actions} + + params = {"_check_parent": parent, "_check_child": child} + ctes = [] + selects = [] + verdicts = {} + + for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): + prefix = f"a{i}_" + rule_parts = [] + restriction_parts = [] + + for permission_sql in permission_sqls: + sql = permission_sql.sql + restriction_sql = permission_sql.restriction_sql + # Namespace this block's params so identical names used for + # different actions cannot collide + for key in permission_sql.params or {}: + new_key = prefix + key + params[new_key] = permission_sql.params[key] + pattern = re.compile(":" + re.escape(key) + r"(?![A-Za-z0-9_])") + if sql: + sql = pattern.sub(":" + new_key, sql) + if restriction_sql: + restriction_sql = pattern.sub(":" + new_key, restriction_sql) + + if restriction_sql: + restriction_parts.append(restriction_sql) + + # Skip plugins that only provide restriction_sql (no permission rules) + if sql is None: + continue + rule_parts.append( + f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" + ) + + if not rule_parts: + # No rules from any plugin - default deny. Restrictions can + # only restrict, never grant, so no SQL is needed at all + verdicts[action] = False + continue + ctes.append(f"a{i}_rules AS (\n" + "\nUNION ALL\n".join(rule_parts) + "\n)") + + # Winning rule for this action: most specific depth first, then + # deny-beats-allow, then source_plugin as a stable tie-break + verdict_sql = f"""COALESCE(( + SELECT allow FROM ( + SELECT allow, source_plugin, + CASE + WHEN child IS NOT NULL THEN 2 + WHEN parent IS NOT NULL THEN 1 + ELSE 0 + END AS depth + FROM a{i}_rules + WHERE (parent IS NULL OR parent = :_check_parent) + AND (child IS NULL OR child = :_check_child) + ORDER BY + depth DESC, + CASE WHEN allow = 0 THEN 0 ELSE 1 END, + source_plugin + LIMIT 1 + ) +), 0)""" + + if restriction_parts: + # Database-level restrictions (parent, NULL) match all children + restriction_intersect = "\nINTERSECT\n".join( + f"SELECT * FROM ({sql})" for sql in restriction_parts + ) + ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)") + verdict_sql = f"""({verdict_sql}) AND EXISTS ( + SELECT 1 FROM a{i}_restriction r + WHERE (r.parent = :_check_parent OR r.parent IS NULL) + AND (r.child = :_check_child OR r.child IS NULL) +)""" + + selects.append(f"SELECT {i} AS action_idx, ({verdict_sql}) AS is_allowed") + + if selects: + query = "WITH\n" + ",\n".join(ctes) + "\n" + "\nUNION ALL\n".join(selects) + result = await datasette.get_internal_database().execute(query, params) + for row in result.rows: + verdicts[unique_actions[row[0]]] = bool(row[1]) + return verdicts + + async def check_permission_for_resource( *, datasette: "Datasette", @@ -515,77 +657,12 @@ async def check_permission_for_resource( Returns: True if the actor is allowed, False otherwise - - This builds the cascading permission query and checks if the specific - resource is in the allowed set. """ - rules_union, all_params, restriction_sqls = await build_permission_rules_sql( - datasette, actor, action + results = await check_permissions_for_actions( + datasette=datasette, + actor=actor, + actions=[action], + parent=parent, + child=child, ) - - # If no rules (empty SQL), default deny - if not rules_union: - return False - - # Add parameters for the resource we're checking - all_params["_check_parent"] = parent - all_params["_check_child"] = child - - # If there are restriction filters, check if the resource passes them first - if restriction_sqls: - # Check if resource is in restriction allowlist - # Database-level restrictions (parent, NULL) should match all children (parent, *) - # Wrap each restriction_sql in a subquery to avoid operator precedence issues - restriction_check = "\nINTERSECT\n".join( - f"SELECT * FROM ({sql})" for sql in restriction_sqls - ) - restriction_query = f""" -WITH restriction_list AS ( - {restriction_check} -) -SELECT EXISTS ( - SELECT 1 FROM restriction_list - WHERE (parent = :_check_parent OR parent IS NULL) - AND (child = :_check_child OR child IS NULL) -) AS in_allowlist -""" - result = await datasette.get_internal_database().execute( - restriction_query, all_params - ) - if result.rows and not result.rows[0][0]: - # Resource not in restriction allowlist - deny - return False - - query = f""" -WITH -all_rules AS ( - {rules_union} -), -matched_rules AS ( - SELECT ar.*, - CASE - WHEN ar.child IS NOT NULL THEN 2 -- child-level (most specific) - WHEN ar.parent IS NOT NULL THEN 1 -- parent-level - ELSE 0 -- root/global - END AS depth - FROM all_rules ar - WHERE (ar.parent IS NULL OR ar.parent = :_check_parent) - AND (ar.child IS NULL OR ar.child = :_check_child) -), -winner AS ( - SELECT * - FROM matched_rules - ORDER BY - depth DESC, -- specificity first (higher depth wins) - CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow - source_plugin -- stable tie-break - LIMIT 1 -) -SELECT COALESCE((SELECT allow FROM winner), 0) AS is_allowed -""" - - # Execute the query against the internal database - result = await datasette.get_internal_database().execute(query, all_params) - if result.rows: - return bool(result.rows[0][0]) - return False + return results[action] diff --git a/docs/internals.rst b/docs/internals.rst index f269155a..f3c1152a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -512,6 +512,39 @@ Example usage: The method returns ``True`` if the permission is granted, ``False`` if denied. +.. _datasette_allowed_many: + +await .allowed_many(\*, actions, resource, actor=None) +------------------------------------------------------ + +``actions`` - list of strings + The names of the actions to permission check. + +``resource`` - Resource object + A Resource object representing the database, table, or other resource that each action is checked against. Omit for global actions. + +``actor`` - dictionary, optional + The authenticated actor. This is usually ``request.actor``. Defaults to ``None`` for unauthenticated requests. + +Checks several actions against the same resource for the same actor, returning a dictionary mapping each action name to ``True`` or ``False``. The whole batch - including any actions pulled in through ``also_requires`` dependencies - is resolved with a single SQL query against the internal database, so this is much faster than calling :ref:`datasette.allowed() ` once per action. + +Example usage: + +.. code-block:: python + + from datasette.resources import TableResource + + results = await datasette.allowed_many( + actions=["insert-row", "delete-row", "drop-table"], + resource=TableResource( + database="fixtures", table="facetable" + ), + actor=request.actor, + ) + # {"insert-row": True, "delete-row": True, "drop-table": False} + +Actions for which no plugin provides any permission rules are resolved to ``False`` directly, without being included in the SQL query at all. + .. _datasette_allowed_resources: await .allowed_resources(action, actor=None, \*, parent=None, include_is_private=False, include_reasons=False, limit=100, next=None) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 2a0ddc93..0a55f8ec 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1458,6 +1458,12 @@ to avoid conflicts with other plugins. The recommended convention is to prefix p plugin's source name (e.g., ``myplugin_user_id``). The system reserves these parameter names: ``:actor``, ``:actor_id``, ``:action``, and ``:filter_parent``. +This hook may be called for many actions in rapid succession - for example +:ref:`datasette.allowed_many() ` gathers rules for every action in its batch +concurrently. Hook implementations must not assume that checks for different actions arrive one +page-render apart, and expensive work (such as network calls) should be cached independently of the +``action`` argument where possible. + You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: .. code-block:: sql diff --git a/tests/conftest.py b/tests/conftest.py index b9b3c35e..27d6fa77 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,6 +146,7 @@ def restore_working_directory(tmpdir, request): @pytest.fixture(scope="session", autouse=True) def check_actions_are_documented(): from datasette.plugins import pm + from datasette.default_actions import register_actions as default_register_actions content = ( pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" @@ -154,6 +155,9 @@ def check_actions_are_documented(): documented_actions = set(permissions_re.findall(content)).union( UNDOCUMENTED_PERMISSIONS ) + # Only Datasette core actions need to be documented - actions registered + # by (test) plugins are checked for registration but not documentation + core_actions = {action.name for action in default_register_actions()} def before(hook_name, hook_impls, kwargs): if hook_name == "permission_resources_sql": @@ -165,9 +169,10 @@ def check_actions_are_documented(): + " (or maybe a test forgot to do await ds.invoke_startup())" ) action = kwargs.get("action").replace("-", "_") - assert ( - action in documented_actions - ), "Undocumented permission action: {}".format(action) + if kwargs["action"] in core_actions: + assert ( + action in documented_actions + ), "Undocumented permission action: {}".format(action) pm.add_hookcall_monitoring( before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py new file mode 100644 index 00000000..53d0ffd9 --- /dev/null +++ b/tests/test_allowed_many.py @@ -0,0 +1,341 @@ +""" +Tests for the datasette.allowed_many() batch permission API, which +resolves multiple actions against one resource in a single internal +database query. datasette.allowed() is implemented on top of it, so +both entry points share one resolution code path. +""" + +import pytest +import pytest_asyncio +from datasette.app import Datasette +from datasette.permissions import PermissionSQL, SkipPermissions +from datasette.resources import DatabaseResource, TableResource +from datasette import hookimpl + + +@pytest_asyncio.fixture +async def ds(): + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("analytics") + await db.execute_write("CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY)") + await db.execute_write("CREATE TABLE IF NOT EXISTS events (id INTEGER PRIMARY KEY)") + await ds._refresh_schemas() + return ds + + +class MatrixRulesPlugin: + """Different rules per action for actor carol, to exercise resolution.""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if not actor or actor.get("id") != "carol": + return None + if action == "view-table": + return PermissionSQL(sql=""" + SELECT NULL AS parent, NULL AS child, 1 AS allow, 'global allow' AS reason + UNION ALL + SELECT 'analytics' AS parent, 'sensitive' AS child, 0 AS allow, 'deny sensitive' AS reason + """) + if action == "insert-row": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'analytics writes' AS reason" + ) + # Everything else: no opinion (implicit deny unless defaults allow) + return None + + +@pytest.mark.asyncio +async def test_allowed_many_basic(ds): + plugin = MatrixRulesPlugin() + ds.pm.register(plugin, name="matrix") + try: + results = await ds.allowed_many( + actions=["view-table", "insert-row", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "carol"}, + ) + assert results == { + "view-table": True, + "insert-row": True, + "drop-table": False, + } + # Child-level deny beats global allow + sensitive = await ds.allowed_many( + actions=["view-table"], + resource=TableResource("analytics", "sensitive"), + actor={"id": "carol"}, + ) + assert sensitive == {"view-table": False} + finally: + ds.pm.unregister(name="matrix") + + +@pytest.mark.asyncio +async def test_allowed_many_matches_allowed(ds): + """Every action resolved by allowed_many() must match allowed().""" + plugin = MatrixRulesPlugin() + ds.pm.register(plugin, name="matrix") + try: + all_actions = list(ds.actions) + for resource in ( + TableResource("analytics", "users"), + TableResource("analytics", "sensitive"), + DatabaseResource("analytics"), + ): + batched = await ds.allowed_many( + actions=all_actions, resource=resource, actor={"id": "carol"} + ) + assert set(batched) == set(all_actions) + for action in all_actions: + individual = await ds.allowed( + action=action, resource=resource, actor={"id": "carol"} + ) + assert ( + batched[action] == individual + ), f"Mismatch for {action} on {resource}" + finally: + ds.pm.unregister(name="matrix") + + +@pytest.mark.asyncio +async def test_allowed_many_unknown_action_raises(ds): + with pytest.raises(ValueError, match="Unknown action"): + await ds.allowed_many( + actions=["view-table", "no-such-action"], + resource=TableResource("analytics", "users"), + actor=None, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_empty_actions(ds): + assert ( + await ds.allowed_many( + actions=[], resource=TableResource("analytics", "users"), actor=None + ) + == {} + ) + + +class AlsoRequiresRulesPlugin: + """dave: store-query allowed but execute-sql explicitly denied. + erin: store-query allowed (execute-sql stays default-allowed).""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + actor_id = actor.get("id") if actor else None + if actor_id == "dave": + if action == "store-query": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'dave can store' AS reason" + ) + if action == "execute-sql": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 0 AS allow, 'dave no sql' AS reason" + ) + if actor_id == "erin" and action == "store-query": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'erin can store' AS reason" + ) + return None + + +@pytest.mark.asyncio +async def test_allowed_many_also_requires(ds): + # store-query also_requires execute-sql, which also_requires view-database + plugin = AlsoRequiresRulesPlugin() + ds.pm.register(plugin, name="also_requires") + try: + resource = DatabaseResource("analytics") + dave = await ds.allowed_many( + actions=["store-query", "execute-sql", "view-database"], + resource=resource, + actor={"id": "dave"}, + ) + # execute-sql denied, so store-query must be denied too + assert dave == { + "store-query": False, + "execute-sql": False, + "view-database": True, + } + erin = await ds.allowed_many( + actions=["store-query"], resource=resource, actor={"id": "erin"} + ) + assert erin == {"store-query": True} + # Must match the single-check path + assert ( + await ds.allowed( + action="store-query", resource=resource, actor={"id": "dave"} + ) + is False + ) + assert ( + await ds.allowed( + action="store-query", resource=resource, actor={"id": "erin"} + ) + is True + ) + finally: + ds.pm.unregister(name="also_requires") + + +@pytest.mark.asyncio +async def test_allowed_many_respects_restrictions(ds): + """Token-style _r restrictions are enforced within the batch.""" + actor = {"id": "root", "_r": {"d": {"analytics": ["vt"]}}} + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor=actor, + ) + # root could normally do both, but the token only allows view-table + # on the analytics database + assert results == {"view-table": True, "drop-table": False} + other_db = await ds.allowed_many( + actions=["view-table"], + resource=TableResource("production", "stuff"), + actor=actor, + ) + assert other_db == {"view-table": False} + # Equivalence with allowed() + assert ( + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "users"), + actor=actor, + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", + resource=TableResource("analytics", "users"), + actor=actor, + ) + is False + ) + + +class ParamCollisionPlugin: + """Same parameter name with a different value for every action.""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if not actor or actor.get("id") != "paula": + return None + flag = 1 if action in ("drop-table", "insert-row") else 0 + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, :flag AS allow, 'flagged' AS reason", + params={"flag": flag}, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_namespaces_params_across_actions(ds): + """Many actions whose rules use identical param names must not collide.""" + plugin = ParamCollisionPlugin() + ds.pm.register(plugin, name="collision") + try: + all_actions = list(ds.actions) + assert len(all_actions) >= 15 + resource = TableResource("analytics", "users") + results = await ds.allowed_many( + actions=all_actions, resource=resource, actor={"id": "paula"} + ) + # Spot-check: only the flagged actions resolve True + assert results["drop-table"] is True + assert results["create-table"] is False + # Full equivalence against single checks + for action in all_actions: + assert results[action] == await ds.allowed( + action=action, resource=resource, actor={"id": "paula"} + ), f"Mismatch for {action}" + finally: + ds.pm.unregister(name="collision") + + +@pytest.mark.asyncio +async def test_allowed_many_single_internal_db_query(ds): + internal_db = ds.get_internal_database() + calls = [] + original_execute = internal_db.execute + + async def counting_execute(sql, params=None, **kwargs): + calls.append(sql) + return await original_execute(sql, params, **kwargs) + + internal_db.execute = counting_execute + try: + results = await ds.allowed_many( + actions=["view-table", "insert-row", "delete-row", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "root", "_r": {"d": {"analytics": ["vt"]}}}, + ) + assert len(results) == 4 + assert len(calls) == 1 + finally: + internal_db.execute = original_execute + + +@pytest.mark.asyncio +async def test_allowed_many_no_query_when_no_rules(ds): + """Actions with no rules from any plugin are denied without SQL. + + Restrictions can only restrict, never grant, so an action with no + rule rows is always False - it should not contribute to the query, + and if no action has rules there should be no query at all.""" + internal_db = ds.get_internal_database() + calls = [] + original_execute = internal_db.execute + + async def counting_execute(sql, params=None, **kwargs): + calls.append(sql) + return await original_execute(sql, params, **kwargs) + + internal_db.execute = counting_execute + try: + # bob gets no rules at all for these write actions + results = await ds.allowed_many( + actions=["drop-table", "delete-row"], + resource=TableResource("analytics", "users"), + actor={"id": "bob"}, + ) + assert results == {"drop-table": False, "delete-row": False} + assert len(calls) == 0 + # A mixed batch still needs exactly one query + calls.clear() + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "bob"}, + ) + assert results == {"view-table": True, "drop-table": False} + assert len(calls) == 1 + finally: + internal_db.execute = original_execute + + +@pytest.mark.asyncio +async def test_allowed_many_global_actions_without_resource(ds): + results = await ds.allowed_many( + actions=["view-instance", "permissions-debug"], + actor={"id": "root"}, + ) + assert results["view-instance"] is True + # Equivalence with single checks for global actions + for action in ("view-instance", "permissions-debug"): + assert results[action] == await ds.allowed(action=action, actor={"id": "root"}) + anon = await ds.allowed_many(actions=["permissions-debug"], actor=None) + assert anon == {"permissions-debug": False} + + +@pytest.mark.asyncio +async def test_allowed_many_skip_permission_checks(ds): + with SkipPermissions(): + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor=None, + ) + assert results == {"view-table": True, "drop-table": True} From bb59c61c9f9b6766199ce1434c7008739653f141 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:11:09 -0700 Subject: [PATCH 1862/1866] Request-scoped permission check cache Adds a per-request cache for permission check results, plus wiring that resolves action permissions in bulk before plugin hooks need them: - New _permission_check_cache contextvar, set to a fresh dict for each request by DatasetteRouter and reset when the request ends. Keys include the full serialized actor, so actors differing in any field (e.g. token restrictions) never share entries. SkipPermissions mode bypasses the cache entirely. - datasette.allowed_many() now consults the cache and stores its results there, so repeated datasette.allowed() checks within one request resolve without further SQL. - Table pages resolve all registered table-level actions against the current table and all database-level actions against its database (database pages likewise) in batched queries before invoking the table_actions/database_actions plugin hooks - allowed() calls made inside those hooks are then served from the cache with no plugin changes required. Actions with no permission rules from any plugin are resolved to False without touching the database. Benchmarks (benchmarks/) with a simulated 12-plugin ecosystem making 18 checks per table page show 34 -> 13 internal-DB queries per page; with 2ms-per-query internal DB latency (modelling Datasette Cloud) table page time drops from 77.9ms to 27.6ms - the caching layer accounts for ~91% of that improvement over allowed_many() alone. Co-Authored-By: Claude Fable 5 --- datasette/app.py | 67 +++++-- datasette/permissions.py | 8 + datasette/views/database.py | 13 ++ datasette/views/table_extras.py | 26 ++- docs/internals.rst | 4 + docs/plugin_hooks.rst | 6 +- tests/test_allowed_many.py | 340 +++++++++++++++++++++++++++++++- 7 files changed, 443 insertions(+), 21 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index a6696ad9..9979b6c5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -291,6 +291,15 @@ DEFAULT_NOT_SET = object() ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) +def _permission_cache_key(actor, action, parent, child): + # Key on the full serialized actor so actors differing in any field + # (e.g. token restrictions) never share cache entries + actor_key = ( + json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None + ) + return (actor_key, action, parent, child) + + async def favicon(request, send): await asgi_send_file( send, @@ -1834,7 +1843,9 @@ class Datasette: Resolves every action (plus any also_requires dependencies) with a single internal database query, instead of one or two queries per - action. + action. Results are stored in the request-scoped permission cache, + so subsequent datasette.allowed() calls for the same checks within + the same request are served from the cache. Example: from datasette.resources import TableResource @@ -1846,6 +1857,10 @@ class Datasette: # {"edit-schema": True, "drop-table": True, "insert-row": False} """ from datasette.utils.actions_sql import check_permissions_for_actions + from datasette.permissions import ( + _permission_check_cache, + _skip_permission_checks, + ) # For global actions, resource is None parent = resource.parent if resource else None @@ -1869,14 +1884,30 @@ class Datasette: for name in requested: add_action(name) - raw = await check_permissions_for_actions( - datasette=self, - actor=actor, - actions=expanded, - parent=parent, - child=child, - ) + # Consult the request-scoped cache, unless permission checks are + # being skipped (skip-mode verdicts must never be cached) + skip = _skip_permission_checks.get() + cache = None if skip else _permission_check_cache.get() + final = {} + to_check = [] + for name in expanded: + if cache is not None: + key = _permission_cache_key(actor, name, parent, child) + if key in cache: + final[name] = cache[key] + continue + to_check.append(name) + + raw = {} + if to_check: + raw = await check_permissions_for_actions( + datasette=self, + actor=actor, + actions=to_check, + parent=parent, + child=child, + ) def resolve(name): # final verdict = own rules AND verdict of also_requires chain @@ -1892,8 +1923,13 @@ class Datasette: for name in expanded: resolve(name) - # Log every check for the debug page, dependencies before the - # actions that required them + # Cache the freshly computed checks + if cache is not None: + for name in to_check: + cache[_permission_cache_key(actor, name, parent, child)] = final[name] + + # Log every check (including cache hits) for the debug page, + # dependencies before the actions that required them when = datetime.datetime.now(datetime.timezone.utc).isoformat() for name in reversed(expanded): self._permission_checks.append( @@ -2663,7 +2699,16 @@ class DatasetteRouter: if raw_path: path = raw_path.decode("ascii") path = path.partition("?")[0] - return await self.route_path(scope, receive, send, path) + # Give each request a fresh permission check cache, so repeated + # datasette.allowed() checks within the request are memoized but + # results never persist beyond it + from datasette.permissions import _permission_check_cache + + cache_token = _permission_check_cache.set({}) + try: + return await self.route_path(scope, receive, send, path) + finally: + _permission_check_cache.reset(cache_token) async def route_path(self, scope, receive, send, path): # Strip off base_url if present before routing diff --git a/datasette/permissions.py b/datasette/permissions.py index a9a3cc7c..786dc026 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -8,6 +8,14 @@ _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False ) +# Request-scoped cache of permission check results. The ASGI router sets +# this to a fresh dict at the start of each request, so cached verdicts +# never outlive a request or leak between actors. Keys are +# (actor_json, action, parent, child) tuples, values are booleans. +_permission_check_cache: contextvars.ContextVar[dict | None] = contextvars.ContextVar( + "permission_check_cache", default=None +) + class SkipPermissions: """Context manager to temporarily skip permission checks. diff --git a/datasette/views/database.py b/datasette/views/database.py index f1756863..6afd9734 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -118,6 +118,19 @@ class DatabaseView(View): ) async def database_actions(): + # Resolve the registered database-level actions for this + # database in one batched query, seeding the request permission + # cache so that allowed() calls made inside the plugin hooks + # below are served from the cache + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database), + actor=request.actor, + ) links = [] for hook in pm.hook.database_actions( datasette=datasette, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 948f3daa..a0308e49 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from datasette.database import QueryInterrupted from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm -from datasette.resources import TableResource +from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, @@ -361,6 +361,30 @@ class ActionsExtra(Extra): else: kwargs["table"] = context.table_name method = pm.hook.table_actions + # Resolve the registered table-level actions for this table + # and the database-level actions for its database in two + # batched queries, seeding the request permission cache so + # that allowed() calls made inside the plugin hooks below + # are served from the cache + datasette = context.datasette + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is TableResource + ], + resource=TableResource(context.database_name, context.table_name), + actor=context.request.actor, + ) + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(context.database_name), + actor=context.request.actor, + ) for hook in method(**kwargs): extra_links = await await_me_maybe(hook) if extra_links: diff --git a/docs/internals.rst b/docs/internals.rst index f3c1152a..641286f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -512,6 +512,8 @@ Example usage: The method returns ``True`` if the permission is granted, ``False`` if denied. +Results are cached for the duration of the current request, so checking the same ``(actor, action, resource)`` combination twice within one request only does the underlying permission resolution work once. + .. _datasette_allowed_many: await .allowed_many(\*, actions, resource, actor=None) @@ -543,6 +545,8 @@ Example usage: ) # {"insert-row": True, "delete-row": True, "drop-table": False} +Each result is stored in the per-request permission check cache, so subsequent ``datasette.allowed()`` calls for the same checks within the same request are served from that cache. Datasette uses this before running the ``table_actions`` and ``database_actions`` plugin hooks: it resolves every registered table-level action against the current table and every database-level action against its database first, which means ``allowed()`` calls made by those plugin hooks are usually served from the cache instead of triggering additional queries. + Actions for which no plugin provides any permission rules are resolved to ``False`` directly, without being included in the SQL query at all. .. _datasette_allowed_resources: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 0a55f8ec..580f7402 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1460,9 +1460,9 @@ plugin's source name (e.g., ``myplugin_user_id``). The system reserves these par This hook may be called for many actions in rapid succession - for example :ref:`datasette.allowed_many() ` gathers rules for every action in its batch -concurrently. Hook implementations must not assume that checks for different actions arrive one -page-render apart, and expensive work (such as network calls) should be cached independently of the -``action`` argument where possible. +concurrently before table and database pages render their action menus. Hook implementations must not +assume that checks for different actions arrive one page-render apart, and expensive work (such as +network calls) should be cached independently of the ``action`` argument where possible. You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 53d0ffd9..3d0d0c9a 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -1,18 +1,52 @@ """ -Tests for the datasette.allowed_many() batch permission API, which -resolves multiple actions against one resource in a single internal -database query. datasette.allowed() is implemented on top of it, so -both entry points share one resolution code path. +Tests for request-scoped permission check memoization and the +datasette.allowed_many() batch permission API. + +Layer 1: per-request cache consulted by datasette.allowed() +Layer 2: allowed_many() resolves multiple actions in one internal-DB query +Layer 3: table/database views precompute all registered actions before + invoking table_actions/database_actions plugin hooks """ import pytest import pytest_asyncio from datasette.app import Datasette -from datasette.permissions import PermissionSQL, SkipPermissions +from datasette.permissions import ( + PermissionSQL, + SkipPermissions, + _permission_check_cache, +) from datasette.resources import DatabaseResource, TableResource from datasette import hookimpl +class CountingRulesPlugin: + """Counts permission_resources_sql gathers and grants rules for alice.""" + + def __init__(self): + self.calls = [] + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + actor_id = actor.get("id") if actor else None + self.calls.append((actor_id, action)) + if actor_id == "alice": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'alice allowed' AS reason" + ) + return None + + def count(self, actor_id=None, action=None): + return len( + [ + (a, c) + for a, c in self.calls + if (actor_id is None or a == actor_id) + and (action is None or c == action) + ] + ) + + @pytest_asyncio.fixture async def ds(): ds = Datasette() @@ -24,6 +58,154 @@ async def ds(): return ds +@pytest_asyncio.fixture +async def counting_ds(ds): + plugin = CountingRulesPlugin() + ds.pm.register(plugin, name="counting") + try: + yield ds, plugin + finally: + ds.pm.unregister(name="counting") + + +# ---------------------------------------------------------------------- +# Layer 1: request-scoped memoization +# ---------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_allowed_memoized_when_cache_active(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + first = await ds.allowed( + action="view-table", resource=resource, actor={"id": "alice"} + ) + gathers_after_first = plugin.count(actor_id="alice", action="view-table") + assert gathers_after_first > 0 + second = await ds.allowed( + action="view-table", resource=resource, actor={"id": "alice"} + ) + assert first is True + assert second is True + # The second identical check must not gather hooks again + assert plugin.count(actor_id="alice", action="view-table") == ( + gathers_after_first + ) + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_allowed_not_memoized_without_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + assert _permission_check_cache.get() is None + await ds.allowed(action="view-table", resource=resource, actor={"id": "alice"}) + first_count = plugin.count(actor_id="alice", action="view-table") + await ds.allowed(action="view-table", resource=resource, actor={"id": "alice"}) + # No request cache active - hooks gathered again + assert plugin.count(actor_id="alice", action="view-table") == first_count * 2 + + +@pytest.mark.asyncio +async def test_cache_keyed_on_full_actor_identity(counting_ds): + """Interleaved checks for different actors never share cache entries.""" + # Uses drop-table because default permissions deny it to non-root actors + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "alice"} + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + # Repeat interleaved - cached results must stay correct per actor + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "alice"} + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + # Actors differing in fields beyond id must not collide either + assert ( + await ds.allowed( + action="drop-table", + resource=resource, + actor={"id": "alice", "_r": {"a": []}}, + ) + is False + ) + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_cache_keyed_on_resource(counting_ds): + ds, plugin = counting_ds + token = _permission_check_cache.set({}) + try: + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "users"), + actor={"id": "alice"}, + ) + count = plugin.count(actor_id="alice", action="view-table") + # Different resource - must not be served from cache + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "events"), + actor={"id": "alice"}, + ) + assert plugin.count(actor_id="alice", action="view-table") == count * 2 + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_skip_permission_checks_bypasses_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + with SkipPermissions(): + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is True + ) + # The skip-mode True must not have been cached + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + finally: + _permission_check_cache.reset(token) + + +# ---------------------------------------------------------------------- +# Layer 2: allowed_many() +# ---------------------------------------------------------------------- + + class MatrixRulesPlugin: """Different rules per action for actor carol, to exercise resolution.""" @@ -233,7 +415,7 @@ class ParamCollisionPlugin: @pytest.mark.asyncio async def test_allowed_many_namespaces_params_across_actions(ds): - """Many actions whose rules use identical param names must not collide.""" + """40+ actions whose rules use identical param names must not collide.""" plugin = ParamCollisionPlugin() ds.pm.register(plugin, name="collision") try: @@ -330,6 +512,24 @@ async def test_allowed_many_global_actions_without_resource(ds): assert anon == {"permissions-debug": False} +@pytest.mark.asyncio +async def test_allowed_many_seeds_request_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + actions = ["view-table", "insert-row", "drop-table"] + token = _permission_check_cache.set({}) + try: + await ds.allowed_many(actions=actions, resource=resource, actor={"id": "alice"}) + gathers = plugin.count(actor_id="alice") + assert gathers > 0 + for action in actions: + await ds.allowed(action=action, resource=resource, actor={"id": "alice"}) + # Every allowed() call must have been served from the seeded cache + assert plugin.count(actor_id="alice") == gathers + finally: + _permission_check_cache.reset(token) + + @pytest.mark.asyncio async def test_allowed_many_skip_permission_checks(ds): with SkipPermissions(): @@ -339,3 +539,131 @@ async def test_allowed_many_skip_permission_checks(ds): actor=None, ) assert results == {"view-table": True, "drop-table": True} + + +# ---------------------------------------------------------------------- +# Layer 3: precompute before table_actions / database_actions hooks +# ---------------------------------------------------------------------- + + +class ActionHooksPlugin: + """Plugin hooks that make allowed() checks, like real action plugins do.""" + + @hookimpl + def table_actions(self, datasette, actor, database, table): + async def inner(): + links = [] + if await datasette.allowed( + action="drop-table", + resource=TableResource(database, table), + actor=actor, + ): + links.append( + {"href": "/drop", "label": "Drop this table (test-plugin)"} + ) + if await datasette.allowed( + action="create-table", + resource=DatabaseResource(database), + actor=actor, + ): + links.append( + {"href": "/create", "label": "Create a table (test-plugin)"} + ) + return links + + return inner + + @hookimpl + def database_actions(self, datasette, actor, database): + async def inner(): + if await datasette.allowed( + action="create-table", + resource=DatabaseResource(database), + actor=actor, + ): + return [{"href": "/create", "label": "Create a table (test-plugin)"}] + return [] + + return inner + + +@pytest_asyncio.fixture +async def spying_ds(ds, monkeypatch): + """ds with the ActionHooksPlugin plus a spy recording every batch of + actions sent to check_permissions_for_actions.""" + from datasette.utils import actions_sql + + plugin = ActionHooksPlugin() + ds.pm.register(plugin, name="action_hooks") + ds.root_enabled = True + recorded = [] + original = actions_sql.check_permissions_for_actions + + async def spy(**kwargs): + recorded.append(kwargs["actions"]) + return await original(**kwargs) + + monkeypatch.setattr(actions_sql, "check_permissions_for_actions", spy) + try: + yield ds, recorded + finally: + ds.pm.unregister(name="action_hooks") + + +@pytest.mark.asyncio +async def test_table_page_precomputes_action_permissions(spying_ds): + ds, recorded = spying_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/analytics/users", cookies=cookies) + assert response.status_code == 200 + # The plugin's permission checks were served from the precomputed batch + assert "Drop this table (test-plugin)" in response.text + assert "Create a table (test-plugin)" in response.text + # One batch covered the table-level actions for the table resource, + # and one covered the database-level actions for the database resource + batches = [batch for batch in recorded if len(batch) > 1] + assert any("drop-table" in batch for batch in batches) + assert any("create-table" in batch for batch in batches) + # The precompute is scoped to actions relevant to each resource: + # no global or query-level actions in any batch, and no mixing of + # table-level and database-level actions + for batch in batches: + assert "view-instance" not in batch + assert "view-query" not in batch + assert not ("drop-table" in batch and "create-table" in batch) + # The hook's own allowed() calls hit the cache - no single-action + # fallback queries for the actions it checked + assert ["drop-table"] not in recorded + assert ["create-table"] not in recorded + + +@pytest.mark.asyncio +async def test_database_page_precomputes_action_permissions(spying_ds): + ds, recorded = spying_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/analytics", cookies=cookies) + assert response.status_code == 200 + assert "Create a table (test-plugin)" in response.text + batches = [batch for batch in recorded if len(batch) > 1] + assert any("create-table" in batch for batch in batches) + # Scoped to database-level actions only + for batch in batches: + assert "view-instance" not in batch + assert "drop-table" not in batch + assert ["create-table"] not in recorded + + +@pytest.mark.asyncio +async def test_cache_does_not_leak_across_requests(counting_ds): + ds, plugin = counting_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "alice"})} + response = await ds.client.get("/analytics/users.json", cookies=cookies) + assert response.status_code == 200 + first_request_gathers = plugin.count(actor_id="alice", action="view-table") + assert first_request_gathers > 0 + response = await ds.client.get("/analytics/users.json", cookies=cookies) + assert response.status_code == 200 + # Second request must re-gather (fresh cache), not reuse the first one + assert ( + plugin.count(actor_id="alice", action="view-table") == first_request_gathers * 2 + ) From d4cb8b464bf1cbe69a8921fc8c9315e04a5f49cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:21:58 -0700 Subject: [PATCH 1863/1866] Fix for trace_child_tasks exception handling I had Claude Fable 5 review our use of contextvar and it spotted this place where exceptions were not correctly handled. --- datasette/tracer.py | 6 ++++-- tests/test_tracer.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 9e66613b..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -27,8 +27,10 @@ def get_task_id(): @contextmanager def trace_child_tasks(): token = trace_task_id.set(get_task_id()) - yield - trace_task_id.reset(token) + try: + yield + finally: + trace_task_id.reset(token) @contextmanager diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6cc80fc4..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -70,6 +70,19 @@ def test_trace_query_errors(): assert trace_info["traces"][-1]["error"] == "no such table: non_existent_table" +@pytest.mark.asyncio +async def test_trace_child_tasks_resets_contextvar_on_exception(): + from datasette import tracer + + before = tracer.trace_task_id.get() + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") + # The contextvar must be reset even though the block raised + assert tracer.trace_task_id.get() == before + + def test_trace_parallel_queries(): with make_app_client(settings={"trace_debug": True}) as client: response = client.get("/parallel-queries?_trace=1") From 86334d233dd2e668169e54fb2312cae2705e7ffc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:09:28 -0700 Subject: [PATCH 1864/1866] Switch to CTE to handle 600+ actions at once GPT-5.5 xhigh in Codex spotted this problem and fixed it with a CTE: https://gisthost.github.io/?46076499ee685acddc988ff6b47a74b0 --- datasette/utils/actions_sql.py | 15 ++++++++++---- tests/test_allowed_many.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index a422c1ed..c7137e6b 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -555,7 +555,7 @@ async def check_permissions_for_actions( params = {"_check_parent": parent, "_check_child": child} ctes = [] - selects = [] + result_rows = [] verdicts = {} for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): @@ -627,10 +627,17 @@ async def check_permissions_for_actions( AND (r.child = :_check_child OR r.child IS NULL) )""" - selects.append(f"SELECT {i} AS action_idx, ({verdict_sql}) AS is_allowed") + result_rows.append(f"({i}, ({verdict_sql}))") - if selects: - query = "WITH\n" + ",\n".join(ctes) + "\n" + "\nUNION ALL\n".join(selects) + if result_rows: + ctes.append( + "results(action_idx, is_allowed) AS (VALUES\n" + + ",\n".join(result_rows) + + "\n)" + ) + query = ( + "WITH\n" + ",\n".join(ctes) + "\nSELECT action_idx, is_allowed FROM results" + ) result = await datasette.get_internal_database().execute(query, params) for row in result.rows: verdicts[unique_actions[row[0]]] = bool(row[1]) diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 3d0d0c9a..08b952fb 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -12,6 +12,7 @@ import pytest import pytest_asyncio from datasette.app import Datasette from datasette.permissions import ( + Action, PermissionSQL, SkipPermissions, _permission_check_cache, @@ -541,6 +542,43 @@ async def test_allowed_many_skip_permission_checks(ds): assert results == {"view-table": True, "drop-table": True} +class ManyActionsPlugin: + """Registers enough actions to exceed SQLite's compound SELECT limit.""" + + def __init__(self, count): + self.action_names = [f"bulk-action-{i}" for i in range(count)] + self.action_names_set = set(self.action_names) + + @hookimpl + def register_actions(self, datasette): + return [ + Action(name=name, abbr=None, description="Bulk test action") + for name in self.action_names + ] + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if action in self.action_names_set: + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'bulk allow' AS reason", + params={}, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_more_than_sqlite_compound_select_limit(): + plugin = ManyActionsPlugin(600) + ds = Datasette() + ds.pm.register(plugin, name="many_actions") + try: + await ds.invoke_startup() + results = await ds.allowed_many(actions=plugin.action_names, actor=None) + assert len(results) == 600 + assert all(results.values()) + finally: + ds.pm.unregister(name="many_actions") + + # ---------------------------------------------------------------------- # Layer 3: precompute before table_actions / database_actions hooks # ---------------------------------------------------------------------- From ab19b0382bc560ef7ae511f899aa7051935577af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:12:31 -0700 Subject: [PATCH 1865/1866] Removed note from permission_resources_sql Refs https://github.com/simonw/datasette/pull/2775/changes#r3408385197 --- docs/plugin_hooks.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 580f7402..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1458,12 +1458,6 @@ to avoid conflicts with other plugins. The recommended convention is to prefix p plugin's source name (e.g., ``myplugin_user_id``). The system reserves these parameter names: ``:actor``, ``:actor_id``, ``:action``, and ``:filter_parent``. -This hook may be called for many actions in rapid succession - for example -:ref:`datasette.allowed_many() ` gathers rules for every action in its batch -concurrently before table and database pages render their action menus. Hook implementations must not -assume that checks for different actions arrive one page-render apart, and expensive work (such as -network calls) should be cached independently of the ``action`` argument where possible. - You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: .. code-block:: sql From f2927a164746a1a2da3e14680948bfbdddfd626b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:15:47 -0700 Subject: [PATCH 1866/1866] Fix for gen.throw(*sys.exc_info()) warning Closes #2776 --- datasette/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 6cd5d11e..e7fe1ed9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -829,10 +829,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): # Execute the actual write try: result = fn(conn) - except Exception: + except Exception as e: # Throw exception into generator so it can handle it try: - gen.throw(*sys.exc_info()) + gen.throw(e) except StopIteration: pass # Re-raise the original exception